From d16544b928a4b00f20fa0bcbe952a1880c67b57a Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 23:56:37 -0500 Subject: [PATCH 01/13] =?UTF-8?q?feat(format):=20a=20filter=20registry=20?= =?UTF-8?q?=E2=80=94=20filters=20are=20looked=20up=20by=20ID?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit decompress_chunk_masked and compress_chunk matched on the filter ID. They now look the ID up in filter_registry: a static table of the built-in filters compiled into this build (a filter whose cargo feature is off is simply absent), then the codecs an application registered at run time with register_filter (a FilterCodec, or a plain decoding closure). Registered codecs cannot shadow a built-in one, and their output is held to the same per-stage bound as the built-in decoders. An ID in neither tier still fails with UnsupportedFilter(id). The "feature off" stub functions that returned UnsupportedFilter are gone: the table leaves those filters out instead. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/filter_registry.rs | 355 ++++++++++++++++++ crates/clawhdf5-format/src/filters.rs | 189 ++++++---- crates/clawhdf5-format/src/filters_szip.rs | 1 + crates/clawhdf5-format/src/lib.rs | 1 + 4 files changed, 464 insertions(+), 82 deletions(-) create mode 100644 crates/clawhdf5-format/src/filter_registry.rs diff --git a/crates/clawhdf5-format/src/filter_registry.rs b/crates/clawhdf5-format/src/filter_registry.rs new file mode 100644 index 0000000..acbf71a --- /dev/null +++ b/crates/clawhdf5-format/src/filter_registry.rs @@ -0,0 +1,355 @@ +//! Filter registry: every filter is looked up here by its HDF5 filter ID. +//! +//! Two tiers: +//! +//! * **Built-in filters** — a static table of the filters compiled into this +//! build: the HDF5 standard filters (deflate, shuffle, Fletcher32, szip, +//! N-Bit, scale-offset) and the plugin filters whose cargo features are +//! enabled (LZ4, Zstandard, pcodec, LZF, bitshuffle, bzip2, blosc). +//! [`builtin_filters`] lists them. +//! * **Registered filters** (`std` only) — codecs the application supplies +//! for any other ID with [`register_filter`] (a [`FilterCodec`], or just a +//! decoding closure). A registered codec cannot shadow a built-in one. +//! +//! An ID in neither tier fails with [`FormatError::UnsupportedFilter`], as it +//! always has. +//! +//! ``` +//! # #[cfg(feature = "std")] { +//! use clawhdf5_format::filter_registry::{self, FilterContext}; +//! use clawhdf5_format::error::FormatError; +//! +//! // A toy filter in the private-use range: every byte XORed with 0x5A. +//! filter_registry::register_filter(300, |input: &[u8], _ctx: &FilterContext<'_>| { +//! Ok::<_, FormatError>(input.iter().map(|b| b ^ 0x5A).collect()) +//! }) +//! .unwrap(); +//! assert!(filter_registry::is_filter_available(300)); +//! filter_registry::unregister_filter(300); +//! # } +//! ``` + +#[cfg(not(feature = "std"))] +extern crate alloc; + +#[cfg(not(feature = "std"))] +use alloc::vec::Vec; + +use crate::error::FormatError; +use crate::filter_pipeline::FilterDescription; + +/// What a codec is told about the filter it is applying. +#[derive(Debug, Clone, Copy)] +pub struct FilterContext<'a> { + /// The filter as recorded in the dataset's filter pipeline: its ID, name, + /// flags and client data (`cd_values`). + pub filter: &'a FilterDescription, + /// Size in bytes of one dataset element (the datatype's size). + pub element_size: usize, + /// Decoding only: the most bytes this stage may produce — what entered + /// the filter when the chunk was written. 0 means unknown; a decoder then + /// falls back to a fixed ceiling. Always 0 when encoding. + pub max_output: usize, +} + +impl FilterContext<'_> { + /// The filter's client data (`cd_values`). + pub fn client_data(&self) -> &[u32] { + &self.filter.client_data + } + + /// The largest output a decoder should allow: [`Self::max_output`], or + /// 256 MiB when that is unknown. + pub fn output_limit(&self) -> usize { + if self.max_output != 0 { + self.max_output + } else { + crate::filters::MAX_DECOMPRESS_SIZE + } + } +} + +/// A filter implementation. +/// +/// `decode` undoes the filter (the read direction). `encode` applies it (the +/// write direction); the default refuses with +/// [`FormatError::UnsupportedFilter`], which is right for a read-only codec. +pub trait FilterCodec: Send + Sync { + /// Undo the filter on one chunk. The output must not exceed + /// [`FilterContext::output_limit`]; the pipeline rejects a larger one. + fn decode(&self, input: &[u8], ctx: &FilterContext<'_>) -> Result, FormatError>; + + /// Apply the filter to one chunk. + fn encode(&self, input: &[u8], ctx: &FilterContext<'_>) -> Result, FormatError> { + let _ = input; + Err(FormatError::UnsupportedFilter(ctx.filter.filter_id)) + } +} + +/// Any `Fn(&[u8], &FilterContext) -> Result, FormatError>` is a +/// decode-only codec. +impl FilterCodec for F +where + F: Fn(&[u8], &FilterContext<'_>) -> Result, FormatError> + Send + Sync, +{ + fn decode(&self, input: &[u8], ctx: &FilterContext<'_>) -> Result, FormatError> { + self(input, ctx) + } +} + +/// Signature of a built-in filter's decoder or encoder. +pub type BuiltinFn = fn(&[u8], &FilterContext<'_>) -> Result, FormatError>; + +/// A filter compiled into this build. +#[derive(Debug, Clone, Copy)] +pub struct BuiltinFilter { + /// HDF5 filter ID. + pub id: u16, + /// Human-readable name. + pub name: &'static str, + /// Decoder. + pub(crate) decode: BuiltinFn, + /// Encoder, if this build can write the filter. + pub(crate) encode: Option, +} + +impl BuiltinFilter { + /// Whether this build can write the filter as well as read it. + pub fn can_encode(&self) -> bool { + self.encode.is_some() + } +} + +/// The filters compiled into this build, in ID order. +pub fn builtin_filters() -> &'static [BuiltinFilter] { + crate::filters::BUILTIN_FILTERS +} + +/// The built-in filter with this ID, if it is compiled in. +pub fn builtin_filter(id: u16) -> Option<&'static BuiltinFilter> { + builtin_filters().iter().find(|f| f.id == id) +} + +/// Whether a chunk filtered with `id` can be decoded: a built-in filter or a +/// registered one. +pub fn is_filter_available(id: u16) -> bool { + if builtin_filter(id).is_some() { + return true; + } + #[cfg(feature = "std")] + { + registered(id).is_some() + } + #[cfg(not(feature = "std"))] + { + false + } +} + +#[cfg(feature = "std")] +mod custom { + use super::FilterCodec; + use std::collections::BTreeMap; + use std::sync::{Arc, PoisonError, RwLock}; + + pub(super) type Registry = BTreeMap>; + + static REGISTRY: RwLock = RwLock::new(BTreeMap::new()); + + pub(super) fn with_read(f: impl FnOnce(&Registry) -> R) -> R { + // A panic while holding the lock cannot leave the map half-updated + // (every update is a single insert/remove), so poisoning is ignored. + f(®ISTRY.read().unwrap_or_else(PoisonError::into_inner)) + } + + pub(super) fn with_write(f: impl FnOnce(&mut Registry) -> R) -> R { + f(&mut REGISTRY.write().unwrap_or_else(PoisonError::into_inner)) + } +} + +/// Register a codec for filter `id`, process-wide. It is used for every +/// chunk read (and, if it implements [`FilterCodec::encode`], written) with +/// that filter ID, by every file. +/// +/// A plain closure `Fn(&[u8], &FilterContext) -> Result, FormatError>` +/// registers a decoder. Replaces (and returns) an earlier registration for +/// the same ID. Fails with [`FormatError::FilterError`] if `id` is a built-in +/// filter of this build: those cannot be overridden. +#[cfg(feature = "std")] +pub fn register_filter( + id: u16, + codec: C, +) -> Result>, FormatError> +where + C: FilterCodec + 'static, +{ + if let Some(builtin) = builtin_filter(id) { + return Err(FormatError::FilterError(format!( + "filter {id} ({}) is built in and cannot be re-registered", + builtin.name + ))); + } + let codec: std::sync::Arc = std::sync::Arc::new(codec); + Ok(custom::with_write(|r| r.insert(id, codec))) +} + +/// Remove the codec registered for `id`. Returns whether one was registered. +#[cfg(feature = "std")] +pub fn unregister_filter(id: u16) -> bool { + custom::with_write(|r| r.remove(&id).is_some()) +} + +/// The codec registered for `id`, if any. +#[cfg(feature = "std")] +pub fn registered(id: u16) -> Option> { + custom::with_read(|r| r.get(&id).cloned()) +} + +/// Undo filter `ctx.filter` on `input`: the built-in decoder if there is one, +/// else a registered one, else [`FormatError::UnsupportedFilter`]. +pub(crate) fn decode(input: &[u8], ctx: &FilterContext<'_>) -> Result, FormatError> { + let id = ctx.filter.filter_id; + if let Some(builtin) = builtin_filter(id) { + return (builtin.decode)(input, ctx); + } + #[cfg(feature = "std")] + if let Some(codec) = registered(id) { + let out = codec.decode(input, ctx)?; + // A registered codec is outside our control: hold it to the same + // bound the built-in decoders enforce. + if out.len() > ctx.output_limit() { + return Err(FormatError::DecompressionError(format!( + "filter {id}: decoded {} bytes, more than the {} the chunk can hold", + out.len(), + ctx.output_limit() + ))); + } + return Ok(out); + } + Err(FormatError::UnsupportedFilter(id)) +} + +/// Apply filter `ctx.filter` to `input`. +pub(crate) fn encode(input: &[u8], ctx: &FilterContext<'_>) -> Result, FormatError> { + let id = ctx.filter.filter_id; + if let Some(builtin) = builtin_filter(id) { + return match builtin.encode { + Some(encode) => encode(input, ctx), + None => Err(FormatError::UnsupportedFilter(id)), + }; + } + #[cfg(feature = "std")] + if let Some(codec) = registered(id) { + return codec.encode(input, ctx); + } + Err(FormatError::UnsupportedFilter(id)) +} + +#[cfg(all(test, feature = "std"))] +mod tests { + use super::*; + use crate::filter_pipeline::{FILTER_FLETCHER32, FILTER_SHUFFLE, FilterPipeline}; + use crate::filters::{compress_chunk, decompress_chunk}; + + fn pipeline(id: u16) -> FilterPipeline { + FilterPipeline { + version: 2, + filters: vec![FilterDescription { + filter_id: id, + name: Some("test".into()), + flags: 0, + client_data: vec![7], + }], + } + } + + struct Xor; + impl FilterCodec for Xor { + fn decode(&self, input: &[u8], ctx: &FilterContext<'_>) -> Result, FormatError> { + let k = ctx.client_data()[0] as u8; + Ok(input.iter().map(|b| b ^ k).collect()) + } + fn encode(&self, input: &[u8], ctx: &FilterContext<'_>) -> Result, FormatError> { + self.decode(input, ctx) + } + } + + // Each test uses its own ID: the registry is process-wide and tests run + // in parallel. + + #[test] + fn unknown_filter_keeps_its_error() { + let err = decompress_chunk(b"abc", &pipeline(311), 3, 1).unwrap_err(); + assert_eq!(err, FormatError::UnsupportedFilter(311)); + let err = compress_chunk(b"abc", &pipeline(311), 1).unwrap_err(); + assert_eq!(err, FormatError::UnsupportedFilter(311)); + } + + #[test] + fn registered_codec_round_trips_through_the_pipeline() { + assert!(!is_filter_available(312)); + assert!(register_filter(312, Xor).unwrap().is_none()); + assert!(is_filter_available(312)); + let data = b"hello, registry".to_vec(); + let enc = compress_chunk(&data, &pipeline(312), 1).unwrap(); + assert_ne!(enc, data); + assert_eq!( + decompress_chunk(&enc, &pipeline(312), data.len(), 1).unwrap(), + data + ); + assert!(unregister_filter(312)); + assert!(!unregister_filter(312)); + assert_eq!( + decompress_chunk(&enc, &pipeline(312), data.len(), 1).unwrap_err(), + FormatError::UnsupportedFilter(312) + ); + } + + #[test] + fn closure_registers_a_decoder_only() { + register_filter(313, |input: &[u8], _ctx: &FilterContext<'_>| { + Ok(input.iter().rev().copied().collect()) + }) + .unwrap(); + assert_eq!( + decompress_chunk(b"abc", &pipeline(313), 3, 1).unwrap(), + b"cba" + ); + assert_eq!( + compress_chunk(b"abc", &pipeline(313), 1).unwrap_err(), + FormatError::UnsupportedFilter(313) + ); + unregister_filter(313); + } + + #[test] + fn registered_decoder_output_is_bounded() { + register_filter(314, |_input: &[u8], _ctx: &FilterContext<'_>| { + Ok(vec![0u8; 1000]) + }) + .unwrap(); + let err = decompress_chunk(b"abc", &pipeline(314), 10, 1).unwrap_err(); + assert!(matches!(err, FormatError::DecompressionError(_)), "{err:?}"); + unregister_filter(314); + } + + #[test] + fn builtins_cannot_be_overridden() { + for id in [FILTER_SHUFFLE, FILTER_FLETCHER32] { + let Err(err) = register_filter(id, Xor) else { + panic!("built-in filter {id} was re-registered"); + }; + assert!(matches!(err, FormatError::FilterError(_)), "{err:?}"); + } + assert!(builtin_filter(FILTER_SHUFFLE).is_some()); + } + + #[test] + fn builtin_table_is_sorted_and_unique() { + let ids: Vec = builtin_filters().iter().map(|f| f.id).collect(); + let mut sorted = ids.clone(); + sorted.sort_unstable(); + sorted.dedup(); + assert_eq!(ids, sorted); + } +} diff --git a/crates/clawhdf5-format/src/filters.rs b/crates/clawhdf5-format/src/filters.rs index 042aca9..e833c7c 100644 --- a/crates/clawhdf5-format/src/filters.rs +++ b/crates/clawhdf5-format/src/filters.rs @@ -7,11 +7,20 @@ extern crate alloc; use alloc::{boxed::Box, vec, vec::Vec}; use crate::error::FormatError; +#[cfg(feature = "deflate")] +use crate::filter_pipeline::FILTER_DEFLATE; +#[cfg(feature = "lz4")] +use crate::filter_pipeline::FILTER_LZ4; +#[cfg(feature = "szip")] +use crate::filter_pipeline::FILTER_SZIP; +#[cfg(feature = "zstd")] +use crate::filter_pipeline::FILTER_ZSTD; use crate::filter_pipeline::{ - FILTER_DEFLATE, FILTER_FLETCHER32, FILTER_LZ4, FILTER_NBIT, FILTER_PCODEC, - FILTER_PCODEC_LEGACY, FILTER_PCODEC_LEGACY_NAME, FILTER_SCALEOFFSET, FILTER_SHUFFLE, - FILTER_SZIP, FILTER_ZSTD, FilterPipeline, + FILTER_FLETCHER32, FILTER_NBIT, FILTER_SCALEOFFSET, FILTER_SHUFFLE, FilterPipeline, }; +#[cfg(feature = "pcodec")] +use crate::filter_pipeline::{FILTER_PCODEC, FILTER_PCODEC_LEGACY, FILTER_PCODEC_LEGACY_NAME}; +use crate::filter_registry::{self, BuiltinFilter, FilterContext}; /// Absolute ceiling on a single decompressed chunk's output size, used only /// when the pipeline's declared `chunk_size` is unavailable (0). Prevents @@ -98,28 +107,14 @@ pub fn decompress_chunk_masked( if filter_skipped(filter_mask, i) { continue; } - let bound = bounds[i]; - data = match filter.filter_id { - FILTER_SHUFFLE => shuffle_decompress(&data, element_size as usize)?, - // `bound` caps the decoded size so these decoders can't be forced - // into unbounded allocation by a hostile or corrupted payload. - FILTER_DEFLATE => deflate_decompress(&data, bound)?, - FILTER_LZ4 => lz4_decompress(&data, bound)?, - FILTER_ZSTD => zstd_decompress(&data, bound)?, - FILTER_FLETCHER32 => fletcher32_verify(&data)?, - FILTER_PCODEC => pcodec_decompress(&data, element_size as usize, bound)?, - // Pcodec chunks written by clawhdf5 <= 2.7.0 under the ID registered - // to Granular BitRound; recognised by the name those versions wrote. - FILTER_PCODEC_LEGACY if filter.name.as_deref() == Some(FILTER_PCODEC_LEGACY_NAME) => { - pcodec_decompress(&data, element_size as usize, bound)? - } - // These decoders also reject an element count that would - // over-allocate past `bound`. - FILTER_SCALEOFFSET => scaleoffset_decompress(&data, &filter.client_data, bound)?, - FILTER_NBIT => nbit_decompress(&data, &filter.client_data, bound)?, - FILTER_SZIP => crate::filters_szip::szip_decompress(&data, &filter.client_data, bound)?, - other => return Err(FormatError::UnsupportedFilter(other)), + // `max_output` caps the decoded size so a decoder can't be forced + // into unbounded allocation by a hostile or corrupted payload. + let ctx = FilterContext { + filter, + element_size: element_size as usize, + max_output: bounds[i], }; + data = filter_registry::decode(&data, &ctx)?; } Ok(data) @@ -135,26 +130,100 @@ pub fn compress_chunk( let mut result = data.to_vec(); for filter in &pipeline.filters { - result = match filter.filter_id { - FILTER_SHUFFLE => shuffle_compress(&result, element_size as usize)?, - FILTER_DEFLATE => { - let level = filter.client_data.first().copied().unwrap_or(6); - deflate_compress(&result, level)? - } - FILTER_LZ4 => lz4_compress(&result, &filter.client_data)?, - FILTER_ZSTD => { - let level = filter.client_data.first().copied().unwrap_or(3); - zstd_compress(&result, level)? - } - FILTER_FLETCHER32 => fletcher32_append(&result)?, - FILTER_PCODEC => pcodec_compress(&result, element_size as usize)?, - other => return Err(FormatError::UnsupportedFilter(other)), + let ctx = FilterContext { + filter, + element_size: element_size as usize, + max_output: 0, }; + result = filter_registry::encode(&result, &ctx)?; } Ok(result) } +/// The filters compiled into this build, sorted by ID (see +/// [`crate::filter_registry`]). A filter whose cargo feature is off is left +/// out, so it fails as [`FormatError::UnsupportedFilter`] like any unknown ID. +pub(crate) static BUILTIN_FILTERS: &[BuiltinFilter] = &[ + #[cfg(feature = "deflate")] + BuiltinFilter { + id: FILTER_DEFLATE, + name: "deflate", + decode: |d, c| deflate_decompress(d, c.max_output), + encode: Some(|d, c| deflate_compress(d, c.client_data().first().copied().unwrap_or(6))), + }, + BuiltinFilter { + id: FILTER_SHUFFLE, + name: "shuffle", + decode: |d, c| shuffle_decompress(d, c.element_size), + encode: Some(|d, c| shuffle_compress(d, c.element_size)), + }, + BuiltinFilter { + id: FILTER_FLETCHER32, + name: "fletcher32", + decode: |d, _| fletcher32_verify(d), + encode: Some(|d, _| fletcher32_append(d)), + }, + #[cfg(feature = "szip")] + BuiltinFilter { + id: FILTER_SZIP, + name: "szip", + decode: |d, c| crate::filters_szip::szip_decompress(d, c.client_data(), c.max_output), + encode: None, + }, + // These decoders also reject an element count that would over-allocate + // past `max_output`. + BuiltinFilter { + id: FILTER_NBIT, + name: "nbit", + decode: |d, c| nbit_decompress(d, c.client_data(), c.max_output), + encode: None, + }, + BuiltinFilter { + id: FILTER_SCALEOFFSET, + name: "scaleoffset", + decode: |d, c| scaleoffset_decompress(d, c.client_data(), c.max_output), + encode: None, + }, + #[cfg(feature = "pcodec")] + BuiltinFilter { + id: FILTER_PCODEC, + name: "pcodec (clawhdf5 private)", + decode: |d, c| pcodec_decompress(d, c.element_size, c.max_output), + encode: Some(|d, c| pcodec_compress(d, c.element_size)), + }, + #[cfg(feature = "lz4")] + BuiltinFilter { + id: FILTER_LZ4, + name: "lz4", + decode: |d, c| lz4_decompress(d, c.max_output), + encode: Some(|d, c| lz4_compress(d, c.client_data())), + }, + #[cfg(feature = "zstd")] + BuiltinFilter { + id: FILTER_ZSTD, + name: "zstd", + decode: |d, c| zstd_decompress(d, c.max_output), + encode: Some(|d, c| zstd_compress(d, c.client_data().first().copied().unwrap_or(3))), + }, + // Pcodec chunks written by clawhdf5 <= 2.7.0 under the ID registered to + // Granular BitRound; recognised by the name those versions wrote, and + // never written. + #[cfg(feature = "pcodec")] + BuiltinFilter { + id: FILTER_PCODEC_LEGACY, + name: "pcodec (clawhdf5 <= 2.7.0)", + decode: |d, c| { + if c.filter.name.as_deref() == Some(FILTER_PCODEC_LEGACY_NAME) { + pcodec_decompress(d, c.element_size, c.max_output) + } else { + Err(FormatError::UnsupportedFilter(FILTER_PCODEC_LEGACY)) + } + }, + encode: None, + }, +]; + /// Decode the HDF5 scale-offset filter (id 6). /// /// Supports all three scale-offset variants: @@ -877,11 +946,6 @@ mod sysz { } } -#[cfg(not(feature = "deflate"))] -fn deflate_decompress(_data: &[u8], _expected_bytes: usize) -> Result, FormatError> { - Err(FormatError::UnsupportedFilter(FILTER_DEFLATE)) -} - /// Compress data with zlib. #[cfg(feature = "deflate")] fn deflate_compress(data: &[u8], level: u32) -> Result, FormatError> { @@ -922,11 +986,6 @@ pub(crate) fn deflate_bounded(data: &[u8], level: u32) -> Result, String } } -#[cfg(not(feature = "deflate"))] -fn deflate_compress(_data: &[u8], _level: u32) -> Result, FormatError> { - Err(FormatError::UnsupportedFilter(FILTER_DEFLATE)) -} - /// Default LZ4 block size of the registered HDF5 LZ4 filter (`H5Zlz4.c`, /// `DEFAULT_BLOCK_SIZE`): 1 GiB, so an HDF5 chunk is normally one block. #[cfg(feature = "lz4")] @@ -1028,11 +1087,6 @@ fn lz4_decompress_hdf5( Ok(out) } -#[cfg(not(feature = "lz4"))] -fn lz4_decompress(_data: &[u8], _expected_bytes: usize) -> Result, FormatError> { - Err(FormatError::UnsupportedFilter(FILTER_LZ4)) -} - /// Compress data in the registered HDF5 LZ4 filter format (see /// [`lz4_decompress`]), so libhdf5 with the LZ4 plugin (e.g. hdf5plugin) can /// read it. `cd[0]`, when present and non-zero, is the block size in bytes, @@ -1064,11 +1118,6 @@ fn lz4_compress(data: &[u8], cd: &[u32]) -> Result, FormatError> { Ok(result) } -#[cfg(not(feature = "lz4"))] -fn lz4_compress(_data: &[u8], _cd: &[u32]) -> Result, FormatError> { - Err(FormatError::UnsupportedFilter(FILTER_LZ4)) -} - /// Decompress zstd data. /// /// `expected_bytes` bounds the output (or [`MAX_DECOMPRESS_SIZE`] when @@ -1097,11 +1146,6 @@ fn zstd_decompress(data: &[u8], expected_bytes: usize) -> Result, Format Ok(out) } -#[cfg(not(feature = "zstd"))] -fn zstd_decompress(_data: &[u8], _expected_bytes: usize) -> Result, FormatError> { - Err(FormatError::UnsupportedFilter(FILTER_ZSTD)) -} - /// Compress data with zstd as one frame whose header records the content /// size. The registered HDF5 Zstandard filter (`H5Zzstd.c`, used by /// libhdf5 + hdf5plugin) sizes its output buffer from @@ -1113,11 +1157,6 @@ fn zstd_compress(data: &[u8], level: u32) -> Result, FormatError> { .map_err(|e| FormatError::CompressionError(format!("zstd: {e}"))) } -#[cfg(not(feature = "zstd"))] -fn zstd_compress(_data: &[u8], _level: u32) -> Result, FormatError> { - Err(FormatError::UnsupportedFilter(FILTER_ZSTD)) -} - /// Unshuffle (decompress direction): reconstruct interleaved element bytes. /// On disk: all byte-0s of each element together, then all byte-1s, etc. /// Output: elements in natural order. @@ -1389,11 +1428,6 @@ fn pcodec_compress(data: &[u8], element_size: usize) -> Result, FormatEr } } -#[cfg(not(feature = "pcodec"))] -fn pcodec_compress(_data: &[u8], _element_size: usize) -> Result, FormatError> { - Err(FormatError::UnsupportedFilter(FILTER_PCODEC)) -} - /// `expected_bytes` bounds the number of elements decoded: the output buffer /// is pre-sized to exactly `expected_bytes / element_size` elements and /// `simple_decompress_into` never writes past it, so a corrupted/hostile pco @@ -1452,15 +1486,6 @@ fn pcodec_decompress( } } -#[cfg(not(feature = "pcodec"))] -fn pcodec_decompress( - _data: &[u8], - _element_size: usize, - _expected_bytes: usize, -) -> Result, FormatError> { - Err(FormatError::UnsupportedFilter(FILTER_PCODEC)) -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/clawhdf5-format/src/filters_szip.rs b/crates/clawhdf5-format/src/filters_szip.rs index 4e7fd71..7e381cf 100644 --- a/crates/clawhdf5-format/src/filters_szip.rs +++ b/crates/clawhdf5-format/src/filters_szip.rs @@ -34,6 +34,7 @@ const SZ_NN_OPTION_MASK: u32 = 32; /// /// The chunk is a 4-byte little-endian uncompressed size followed by the /// szlib stream. +#[cfg_attr(not(feature = "szip"), allow(dead_code))] pub(crate) fn szip_decompress( _data: &[u8], _cd: &[u32], diff --git a/crates/clawhdf5-format/src/lib.rs b/crates/clawhdf5-format/src/lib.rs index ef09c1b..74573c0 100644 --- a/crates/clawhdf5-format/src/lib.rs +++ b/crates/clawhdf5-format/src/lib.rs @@ -71,6 +71,7 @@ pub mod extensible_array; pub mod file_writer; pub mod fill_value; pub mod filter_pipeline; +pub mod filter_registry; pub mod filters; mod filters_szip; pub mod fixed_array; From e38f9123dbba043ecaa7bd423be4a39e9b54abae Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 00:01:22 -0500 Subject: [PATCH 02/13] feat(format): LZF filter (32000), read and write, pure Rust h5py's built-in compression="lzf" failed with UnsupportedFilter(32000). The new `lzf` feature (no dependencies, on by default in clawhdf5-format and the facade) decodes the raw liblzf stream h5py's filter stores, bounded by the chunk size, and encodes it: DatasetBuilder::with_lzf() (or with_plugin_filter(PluginFilter::Lzf)) writes the filter with h5py's cd_values (filter version 4, liblzf 0x0105, chunk size in bytes), flagged optional as h5py does. ChunkOptions gains a `plugin` field for the plugin filters; build_pipeline_for_chunk passes the chunk size to filters that record it. tests/plugin_filters_interop.rs: h5py writes LZF (alone, with shuffle, with shuffle+fletcher32) over 12 dtype/shape/chunk/data cases with partial edge chunks and incompressible data, and every dataset reads byte for byte equal to its unfiltered twin; our LZF output (1-D and 2-D, edge chunks, with and without shuffle) reads back in h5py. Both fail with the decoder removed. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/Cargo.toml | 5 +- crates/clawhdf5-format/src/chunked_write.rs | 92 +++++- crates/clawhdf5-format/src/filter_pipeline.rs | 12 + crates/clawhdf5-format/src/filters.rs | 9 +- crates/clawhdf5-format/src/filters_lzf.rs | 232 +++++++++++++++ crates/clawhdf5-format/src/lib.rs | 2 + crates/clawhdf5-format/src/type_builders.rs | 18 ++ crates/clawhdf5/Cargo.toml | 5 +- .../clawhdf5/tests/plugin_filters_interop.rs | 263 ++++++++++++++++++ 9 files changed, 629 insertions(+), 9 deletions(-) create mode 100644 crates/clawhdf5-format/src/filters_lzf.rs create mode 100644 crates/clawhdf5/tests/plugin_filters_interop.rs diff --git a/crates/clawhdf5-format/Cargo.toml b/crates/clawhdf5-format/Cargo.toml index a88b734..4eb4c21 100644 --- a/crates/clawhdf5-format/Cargo.toml +++ b/crates/clawhdf5-format/Cargo.toml @@ -37,7 +37,7 @@ harness = false # Deflate backend: `zlib-rs` (pure Rust) by default. `fast-deflate` selects # zlib-ng instead (C, built with cmake); flate2 prefers a C zlib whenever one # is enabled, so turning it on anywhere in the build overrides the default. -default = ["std", "checksum", "deflate", "provenance", "zlib-rs", "system-zlib-decompress"] +default = ["std", "checksum", "deflate", "provenance", "zlib-rs", "system-zlib-decompress", "lzf"] std = [] checksum = [] deflate = ["flate2"] @@ -56,6 +56,9 @@ zstd = ["dep:zstd"] blake3_hash = ["blake3"] szip = ["libaec-sys"] pcodec = ["dep:pco"] +# Plugin filters, pure Rust. LZF (32000) is h5py's built-in compression; it +# has no dependencies, so it is on by default. +lzf = [] [[bench]] name = "parallel_decompress_bench" diff --git a/crates/clawhdf5-format/src/chunked_write.rs b/crates/clawhdf5-format/src/chunked_write.rs index ff5081e..4e9b851 100644 --- a/crates/clawhdf5-format/src/chunked_write.rs +++ b/crates/clawhdf5-format/src/chunked_write.rs @@ -12,7 +12,7 @@ use crate::chunk_grid::ChunkGrid; use crate::ea_writer; use crate::error::FormatError; use crate::filter_pipeline::{ - FILTER_DEFLATE, FILTER_FLETCHER32, FILTER_LZ4, FILTER_PCODEC, FILTER_PCODEC_NAME, + FILTER_DEFLATE, FILTER_FLETCHER32, FILTER_LZ4, FILTER_LZF, FILTER_PCODEC, FILTER_PCODEC_NAME, FILTER_SHUFFLE, FILTER_ZSTD, FilterDescription, FilterPipeline, }; use crate::filters::compress_chunk; @@ -48,6 +48,45 @@ pub struct ChunkOptions { /// Pcodec lossless numerical compression. Private, unregistered filter /// ID [`FILTER_PCODEC`] (480): only clawhdf5 can read it. pub pcodec: bool, + /// A plugin compression filter (LZF, ...). Takes priority over the + /// codecs above. Each needs its cargo feature to be written. + pub plugin: Option, +} + +/// A compression filter from the common HDF5 plugin set, written in the +/// format the libhdf5 plugin (h5py / hdf5plugin) reads. +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub enum PluginFilter { + /// LZF (filter 32000), h5py's built-in `compression="lzf"`. Needs the + /// `lzf` feature. + Lzf, +} + +impl PluginFilter { + /// Whether the filter reorders bytes itself, so the automatic shuffle + /// pre-filter would only get in its way. + fn shuffles_itself(&self) -> bool { + match self { + PluginFilter::Lzf => false, + } + } + + /// The pipeline entry for this filter. `chunk_bytes` is one chunk's + /// uncompressed size (0 if unknown). + fn description(&self, _element_size: u32, chunk_bytes: u32) -> FilterDescription { + match self { + // h5py's lzf_set_local: filter version, liblzf version, chunk + // size in bytes. Optional, as h5py flags it: a chunk the filter + // cannot shrink may then be stored unfiltered. + PluginFilter::Lzf => FilterDescription { + filter_id: FILTER_LZF, + name: Some("lzf".into()), + flags: 1, + client_data: vec![4, 0x0105, chunk_bytes], + }, + } + } } /// Largest chunk the automatic choice produces, in bytes. @@ -92,14 +131,33 @@ impl ChunkOptions { || self.lz4 || self.zstd_level.is_some() || self.pcodec + || self.plugin.is_some() } /// Build a FilterPipeline from the options. pub fn build_pipeline(&self, element_size: u32) -> Option { + self.build_pipeline_for_chunk(element_size, 0) + } + + /// Build a FilterPipeline for chunks of `chunk_bytes` uncompressed bytes + /// (0 if unknown). Some plugin filters record the chunk size in their + /// client data. + pub fn build_pipeline_for_chunk( + &self, + element_size: u32, + chunk_bytes: u32, + ) -> Option { let mut filters = Vec::new(); - let has_compression = - self.deflate_level.is_some() || self.zstd_level.is_some() || self.lz4 || self.pcodec; + let plugin_shuffles = self + .plugin + .as_ref() + .is_some_and(PluginFilter::shuffles_itself); + let has_compression = self.deflate_level.is_some() + || self.zstd_level.is_some() + || self.lz4 + || self.pcodec + || (self.plugin.is_some() && !plugin_shuffles); // Shuffle before compression. Applied if explicitly requested OR if compression // is active and the caller hasn't disabled it — matches h5py default behavior @@ -113,8 +171,11 @@ impl ChunkOptions { }); } - // Compression filters (mutually exclusive, priority: pcodec > zstd > lz4 > deflate) - if self.pcodec { + // Compression filters (mutually exclusive, priority: plugin > pcodec > + // zstd > lz4 > deflate) + if let Some(plugin) = &self.plugin { + filters.push(plugin.description(element_size, chunk_bytes)); + } else if self.pcodec { filters.push(FilterDescription { filter_id: FILTER_PCODEC, name: Some(FILTER_PCODEC_NAME.into()), @@ -675,7 +736,12 @@ pub fn precompress_chunks( element_size: usize, options: &ChunkOptions, ) -> Result { - let pipeline = options.build_pipeline(element_size as u32); + let chunk_bytes = chunk_dims + .iter() + .try_fold(element_size as u64, |acc, &d| acc.checked_mul(d)) + .and_then(|b| u32::try_from(b).ok()) + .unwrap_or(0); + let pipeline = options.build_pipeline_for_chunk(element_size as u32, chunk_bytes); let has_filters = pipeline.is_some(); let pipeline_message = pipeline.as_ref().map(|pl| pl.serialize()); @@ -1569,6 +1635,20 @@ mod tests { assert_eq!(pl.filters[1].client_data, vec![3]); } + #[test] + fn chunk_options_pipeline_lzf() { + let options = ChunkOptions { + plugin: Some(PluginFilter::Lzf), + ..Default::default() + }; + assert!(options.is_chunked()); + let pl = options.build_pipeline_for_chunk(8, 800).unwrap(); + assert_eq!(pl.filters.len(), 2); + assert_eq!(pl.filters[0].filter_id, FILTER_SHUFFLE); + assert_eq!(pl.filters[1].filter_id, FILTER_LZF); + assert_eq!(pl.filters[1].client_data, vec![4, 0x0105, 800]); + } + #[test] fn chunk_options_zstd_priority_over_deflate() { let options = ChunkOptions { diff --git a/crates/clawhdf5-format/src/filter_pipeline.rs b/crates/clawhdf5-format/src/filter_pipeline.rs index 74bc727..13ed69f 100644 --- a/crates/clawhdf5-format/src/filter_pipeline.rs +++ b/crates/clawhdf5-format/src/filter_pipeline.rs @@ -19,6 +19,18 @@ pub const FILTER_SCALEOFFSET: u16 = 6; pub const FILTER_LZ4: u16 = 32004; /// Zstandard compression. pub const FILTER_ZSTD: u16 = 32015; +/// bzip2 (registered by PyTables; hdf5plugin's `BZip2`). +pub const FILTER_BZIP2: u16 = 307; +/// LZF — h5py's built-in `compression="lzf"`. +pub const FILTER_LZF: u16 = 32000; +/// Blosc 1 (hdf5-blosc; hdf5plugin's `Blosc`). +pub const FILTER_BLOSC: u16 = 32001; +/// Bitshuffle, optionally with LZ4 or Zstandard (hdf5plugin's `Bitshuffle`). +pub const FILTER_BITSHUFFLE: u16 = 32008; +/// ZFP lossy floating-point compression (hdf5plugin's `Zfp`). Not supported. +pub const FILTER_ZFP: u16 = 32013; +/// Blosc 2 (hdf5plugin's `Blosc2`). +pub const FILTER_BLOSC2: u16 = 32026; /// Pcodec lossless numerical codec — a **private, unregistered** clawhdf5 /// filter. Pcodec has no ID in the HDF Group's filter registry (checked /// 2026-09-25, `hdf5_plugins/docs/RegisteredFilterPlugins.md`), so it uses an diff --git a/crates/clawhdf5-format/src/filters.rs b/crates/clawhdf5-format/src/filters.rs index e833c7c..237ceb5 100644 --- a/crates/clawhdf5-format/src/filters.rs +++ b/crates/clawhdf5-format/src/filters.rs @@ -192,6 +192,13 @@ pub(crate) static BUILTIN_FILTERS: &[BuiltinFilter] = &[ decode: |d, c| pcodec_decompress(d, c.element_size, c.max_output), encode: Some(|d, c| pcodec_compress(d, c.element_size)), }, + #[cfg(feature = "lzf")] + BuiltinFilter { + id: crate::filter_pipeline::FILTER_LZF, + name: "lzf", + decode: crate::filters_lzf::lzf_decode, + encode: Some(crate::filters_lzf::lzf_encode), + }, #[cfg(feature = "lz4")] BuiltinFilter { id: FILTER_LZ4, @@ -1702,7 +1709,7 @@ mod tests { // An unsupported filter is fine when the chunk skipped it. let unknown = FilterPipeline { version: 2, - filters: vec![filter(32000), filter(FILTER_DEFLATE)], + filters: vec![filter(32013), filter(FILTER_DEFLATE)], }; assert_eq!( decompress_chunk_masked(&deflated, &unknown, n, 8, 0b01).unwrap(), diff --git a/crates/clawhdf5-format/src/filters_lzf.rs b/crates/clawhdf5-format/src/filters_lzf.rs new file mode 100644 index 0000000..5657339 --- /dev/null +++ b/crates/clawhdf5-format/src/filters_lzf.rs @@ -0,0 +1,232 @@ +//! LZF (HDF5 filter 32000) — h5py's built-in compression filter +//! (`compression="lzf"`), in pure Rust. +//! +//! The chunk is one raw LZF stream (liblzf 3.x format, no header). The +//! stream is a sequence of instructions, each starting with a control byte: +//! +//! * `000LLLLL` — a literal run: the next `L + 1` bytes (1..=32) are copied. +//! * `LLLOOOOO [E] OOOOOOOO` — a back reference: copy `len + 2` bytes from +//! `distance` bytes back, where `len` is the top three bits (1..=6), or +//! `7 + E` when they are all ones, and `distance` is the 13-bit offset +//! (high five bits in the control byte, low eight in the last byte) plus 1. +//! +//! h5py's filter (`lzf_filter.c`) records the chunk's size in bytes in +//! `cd_values[2]` (slots 0 and 1 hold the filter and liblzf versions) and +//! sizes its output buffer from it. + +#[cfg(not(feature = "std"))] +extern crate alloc; +#[cfg(not(feature = "std"))] +use alloc::{format, vec, vec::Vec}; + +use crate::error::FormatError; +use crate::filter_registry::FilterContext; + +/// `H5PY_FILTER_LZF_VERSION`, written to `cd_values[0]`. +pub const LZF_FILTER_VERSION: u32 = 4; +/// `LZF_VERSION` (liblzf 1.5), written to `cd_values[1]`. +pub const LZF_API_VERSION: u32 = 0x0105; + +const MAX_LITERAL: usize = 32; +const MAX_OFFSET: usize = 1 << 13; +const MAX_REF: usize = (1 << 8) + (1 << 3); +const HASH_LOG: u32 = 14; + +fn err(msg: &str) -> FormatError { + FormatError::DecompressionError(format!("lzf: {msg}")) +} + +/// Decode an LZF-filtered chunk. +pub(crate) fn lzf_decode(input: &[u8], ctx: &FilterContext<'_>) -> Result, FormatError> { + let limit = ctx.output_limit(); + let hint = match ctx.client_data().get(2) { + Some(&n) if n != 0 => n as usize, + _ => input.len().saturating_mul(2), + }; + lzf_decompress(input, hint.min(limit), limit) +} + +/// Decompress a raw LZF stream, refusing to produce more than `limit` bytes. +pub fn lzf_decompress( + input: &[u8], + size_hint: usize, + limit: usize, +) -> Result, FormatError> { + let mut out: Vec = Vec::new(); + out.try_reserve(size_hint) + .map_err(|_| err("cannot allocate the output buffer"))?; + let mut ip = 0usize; + while ip < input.len() { + let ctrl = input[ip] as usize; + ip += 1; + if ctrl < 32 { + let run = ctrl + 1; + let lit = input + .get(ip..ip + run) + .ok_or_else(|| err("literal run past the end of the input"))?; + if out.len() + run > limit { + return Err(err("output exceeds the chunk size")); + } + out.extend_from_slice(lit); + ip += run; + } else { + let mut len = ctrl >> 5; + if len == 7 { + len += *input + .get(ip) + .ok_or_else(|| err("truncated back reference"))? + as usize; + ip += 1; + } + let low = *input + .get(ip) + .ok_or_else(|| err("truncated back reference"))? as usize; + ip += 1; + let distance = ((ctrl & 0x1f) << 8) + low + 1; + let len = len + 2; + if distance > out.len() { + return Err(err("back reference before the start of the output")); + } + if out.len() + len > limit { + return Err(err("output exceeds the chunk size")); + } + let start = out.len() - distance; + if distance >= len { + out.extend_from_within(start..start + len); + } else { + // Overlapping copy: repeats the last `distance` bytes. + for k in 0..len { + let b = out[start + k]; + out.push(b); + } + } + } + } + Ok(out) +} + +/// Encode a chunk with the LZF filter. +pub(crate) fn lzf_encode(input: &[u8], _ctx: &FilterContext<'_>) -> Result, FormatError> { + Ok(lzf_compress(input)) +} + +fn hash3(b: &[u8]) -> usize { + let v = (u32::from(b[0]) << 16) | (u32::from(b[1]) << 8) | u32::from(b[2]); + (v.wrapping_mul(2_654_435_761) >> (32 - HASH_LOG)) as usize +} + +fn flush_literals(out: &mut Vec, lit: &[u8]) { + for run in lit.chunks(MAX_LITERAL) { + out.push((run.len() - 1) as u8); + out.extend_from_slice(run); + } +} + +/// Compress `input` into a raw LZF stream any liblzf decoder reads. +/// +/// Incompressible input grows by one byte per 32. (h5py's own filter gives +/// up on such a chunk and stores it unfiltered; storing the slightly larger +/// stream is equally readable.) +pub fn lzf_compress(input: &[u8]) -> Vec { + let n = input.len(); + let mut out = Vec::with_capacity(n + n / MAX_LITERAL + 1); + let mut table = vec![0u32; 1 << HASH_LOG]; + let mut lit_start = 0usize; + let mut i = 0usize; + while i + 2 < n { + let h = hash3(&input[i..]); + let cand = table[h] as usize; + table[h] = (i + 1) as u32; + if cand != 0 { + let r = cand - 1; + let distance = i - r; + if distance <= MAX_OFFSET && input[r..r + 3] == input[i..i + 3] { + let max_len = (n - i).min(MAX_REF); + let mut len = 3; + while len < max_len && input[r + len] == input[i + len] { + len += 1; + } + flush_literals(&mut out, &input[lit_start..i]); + let code = len - 2; + let off = distance - 1; + if code < 7 { + out.push(((code << 5) | (off >> 8)) as u8); + } else { + out.push(((7 << 5) | (off >> 8)) as u8); + out.push((code - 7) as u8); + } + out.push((off & 0xff) as u8); + // Index the positions the match covered so later data can + // refer back into it. + let end = i + len; + let mut j = i + 1; + while j < end && j + 2 < n { + table[hash3(&input[j..])] = (j + 1) as u32; + j += 1; + } + i = end; + lit_start = i; + continue; + } + } + i += 1; + } + flush_literals(&mut out, &input[lit_start..]); + out +} + +#[cfg(test)] +mod tests { + use super::*; + + fn round_trip(data: &[u8]) { + let c = lzf_compress(data); + assert_eq!(lzf_decompress(&c, data.len(), data.len()).unwrap(), data); + } + + #[test] + fn round_trips() { + round_trip(b""); + round_trip(b"a"); + round_trip(b"abcabcabcabcabcabcabcabcabcabcabcabc"); + round_trip(&[7u8; 10_000]); + let noise: Vec = (0..70_000u32) + .map(|i| (i.wrapping_mul(2_654_435_761) >> 13) as u8) + .collect(); + round_trip(&noise); + let ramp: Vec = (0..100_000u32) + .flat_map(|i| (i % 1000).to_le_bytes()) + .collect(); + round_trip(&ramp); + } + + #[test] + fn compresses_repetitive_data() { + let data = [42u8; 4096]; + assert!(lzf_compress(&data).len() < 100); + } + + /// The chunk h5py 3.16's bundled liblzf writes for + /// `b"hello hello hello hello"` (read back with `read_direct_chunk`): a + /// 7-byte literal, a 14-byte back reference 6 bytes back (extended + /// length), and a 2-byte literal. + #[test] + fn decodes_liblzf_output() { + let stream = b"\x06hello h\xe0\x05\x05\x01lo"; + assert_eq!( + lzf_decompress(stream, 23, 23).unwrap(), + b"hello hello hello hello" + ); + } + + #[test] + fn rejects_corrupt_streams() { + // Back reference before the start. + assert!(lzf_decompress(&[0x20, 0x00], 10, 10).is_err()); + // Literal run past the end. + assert!(lzf_decompress(&[0x05, 1, 2], 10, 10).is_err()); + // Output over the limit. + let c = lzf_compress(&[1u8; 100]); + assert!(lzf_decompress(&c, 10, 99).is_err()); + } +} diff --git a/crates/clawhdf5-format/src/lib.rs b/crates/clawhdf5-format/src/lib.rs index 74573c0..c717d89 100644 --- a/crates/clawhdf5-format/src/lib.rs +++ b/crates/clawhdf5-format/src/lib.rs @@ -73,6 +73,8 @@ pub mod fill_value; pub mod filter_pipeline; pub mod filter_registry; pub mod filters; +#[cfg(feature = "lzf")] +pub mod filters_lzf; mod filters_szip; pub mod fixed_array; pub mod float16; diff --git a/crates/clawhdf5-format/src/type_builders.rs b/crates/clawhdf5-format/src/type_builders.rs index 9e46e66..c49ccc7 100644 --- a/crates/clawhdf5-format/src/type_builders.rs +++ b/crates/clawhdf5-format/src/type_builders.rs @@ -731,6 +731,24 @@ impl DatasetBuilder { self } + /// Compress with a plugin filter ([`PluginFilter`]), in the format the + /// libhdf5 plugin reads (h5py, hdf5plugin). Implies chunked storage. + /// Each filter needs its cargo feature (`lzf`, ...); writing fails with + /// `UnsupportedFilter` without it. + /// + /// [`PluginFilter`]: crate::chunked_write::PluginFilter + pub fn with_plugin_filter(&mut self, filter: crate::chunked_write::PluginFilter) -> &mut Self { + self.chunk_options.plugin = Some(filter); + self + } + + /// Enable LZF compression (filter 32000) — h5py's built-in + /// `compression="lzf"`. Implies chunked storage; shuffle is applied + /// first unless `.without_shuffle()`. Requires the `lzf` cargo feature. + pub fn with_lzf(&mut self) -> &mut Self { + self.with_plugin_filter(crate::chunked_write::PluginFilter::Lzf) + } + /// Enable Pcodec lossless numerical compression (private clawhdf5 filter /// ID 480). /// diff --git a/crates/clawhdf5/Cargo.toml b/crates/clawhdf5/Cargo.toml index 2e2b4b4..f4c6ee2 100644 --- a/crates/clawhdf5/Cargo.toml +++ b/crates/clawhdf5/Cargo.toml @@ -31,7 +31,7 @@ name = "parallel_bench" harness = false [features] -default = ["mmap", "provenance"] +default = ["mmap", "provenance", "lzf"] mmap = ["clawhdf5-io/mmap"] parallel = ["clawhdf5-format/parallel", "rayon"] # zlib-ng (C, needs cmake) instead of the default pure-Rust zlib-rs. @@ -41,6 +41,9 @@ zstd = ["clawhdf5-format/zstd"] blake3_hash = ["clawhdf5-format/blake3_hash"] lz4 = ["clawhdf5-format/lz4"] pcodec = ["clawhdf5-format/pcodec"] +# Plugin filters, pure Rust (no C). LZF (32000) is h5py's built-in +# compression; it has no dependencies, so it is on by default. +lzf = ["clawhdf5-format/lzf"] # Dataset::verify_provenance() — recompute a dataset's SHA-256 and compare # against its stored _provenance_sha256 attribute. On by default, matching # clawhdf5-format's own default-on `provenance` feature. diff --git a/crates/clawhdf5/tests/plugin_filters_interop.rs b/crates/clawhdf5/tests/plugin_filters_interop.rs new file mode 100644 index 0000000..041aff1 --- /dev/null +++ b/crates/clawhdf5/tests/plugin_filters_interop.rs @@ -0,0 +1,263 @@ +//! Plugin filters (LZF, bitshuffle, bzip2, blosc) against libhdf5. +//! +//! Read direction: h5py (with hdf5plugin for everything but LZF, which h5py +//! ships) writes each filter over a matrix of dtypes (1-8 bytes, both byte +//! orders), 1-3 dimensional shapes whose chunks do not divide them (so the +//! edge chunks are partial), compressible and incompressible data, and the +//! filter's own options; every dataset has an unfiltered twin, and clawhdf5 +//! must read the filtered one byte for byte equal to it. +//! +//! Write direction: clawhdf5 writes with its encoder, and h5py must read the +//! values back. +//! +//! Skipped when python3 with h5py (and hdf5plugin) is unavailable, unless +//! `CLAWHDF5_REQUIRE_INTEROP=1`. +#![allow(dead_code)] + +use std::process::Command; + +use clawhdf5::File; +use clawhdf5_format::selection::Selection; + +fn python() -> String { + std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) +} + +fn interop_required() -> bool { + std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1") +} + +fn python_has(modules: &str) -> bool { + Command::new(python()) + .args(["-c", &format!("import {modules}")]) + .output() + .map(|o| o.status.success()) + .unwrap_or(false) +} + +/// Whether the interop test can run; panics instead of skipping when +/// `CLAWHDF5_REQUIRE_INTEROP=1`. +fn have_python(modules: &str) -> bool { + if python_has(modules) { + return true; + } + assert!( + !interop_required(), + "CLAWHDF5_REQUIRE_INTEROP=1 but python3 with {modules} is not available" + ); + eprintln!("SKIP: python3 with {modules} not available"); + false +} + +fn run_python(script: &str, args: &[&str]) -> String { + let output = Command::new(python()) + .arg("-c") + .arg(script) + .args(args) + .output() + .expect("failed to run python"); + if !output.status.success() { + panic!( + "Python script failed:\nSTDOUT: {}\nSTDERR: {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } + String::from_utf8_lossy(&output.stdout).trim().to_string() +} + +/// Writes `f{i}` (filtered) and `r{i}` (unfiltered twin) for every filter +/// setting in `FILTERS` × every case; prints the number of pairs. +const GENERATE: &str = r#" +import sys +import numpy as np, h5py +try: + import hdf5plugin +except ImportError: + hdf5plugin = None +path = sys.argv[1] +FILTERS = eval(sys.argv[2]) +cases = [ + ('i4', (37, 53), (8, 8), 'ramp'), + (' 0); + let file = File::open(&path).unwrap(); + for i in 0..n { + let filtered = file.dataset(&format!("f{i}")).unwrap(); + let case = format!("{:?}", filtered.attrs().unwrap().get("case")); + let got = filtered + .read_selection(&Selection::All) + .unwrap_or_else(|e| panic!("{tag} f{i} {case}: {e}")); + let want = file + .dataset(&format!("r{i}")) + .unwrap() + .read_selection(&Selection::All) + .unwrap(); + assert!(got == want, "{tag} f{i} {case}: data differs"); + } +} + +/// Values the write-direction tests store: row-major, compressible with some +/// variation. +fn ramp_i32(n: usize) -> Vec { + (0..n).map(|i| ((i * 3) % 251) as i32 - 60).collect() +} +fn ramp_f64(n: usize) -> Vec { + (0..n).map(|i| ((i * 3) % 251) as f64 / 7.0).collect() +} +fn ramp_u8(n: usize) -> Vec { + (0..n).map(|i| ((i * 7) % 256) as u8).collect() +} + +/// h5py checks the datasets `write_ours` wrote against the same ramps. +const VERIFY: &str = r#" +import sys +import numpy as np, h5py +try: + import hdf5plugin +except ImportError: + pass +bad = [] +with h5py.File(sys.argv[1], 'r') as f: + for name in f: + ds = f[name] + n = ds.size + k = np.arange(n) + if ds.dtype == np.int32: + want = ((k * 3) % 251 - 60).astype(np.int32) + elif ds.dtype == np.float64: + want = ((k * 3) % 251) / 7.0 + else: + want = ((k * 7) % 256).astype(np.uint8) + got = ds[()].reshape(-1) + if not np.array_equal(got, want): + bad.append(name) + # The dataset really is filtered by the plugin under test. + if int(sys.argv[2]) not in [int(x) for x in ds._filters.keys() if x.isdigit()] \ + and sys.argv[3] not in ds._filters: + bad.append(name + ':filter-missing:' + repr(ds._filters)) +print('OK' if not bad else 'BAD ' + ' '.join(bad)) +"#; + +/// Write i32/f64/u8 datasets (1-D and 2-D, partial edge chunks) with +/// `configure` applying the filter, then have h5py read them back. +fn check_ours_read_by_h5py( + tag: &str, + filter_id: u16, + filter_name: &str, + configure: impl Fn(&mut clawhdf5_format::type_builders::DatasetBuilder), +) { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join(format!("{tag}_ours.h5")); + let mut fb = clawhdf5::FileBuilder::new(); + { + let ds = fb.create_dataset("i32_1d"); + ds.with_i32_data(&ramp_i32(10_000)).with_chunks(&[3000]); + configure(ds); + } + { + let ds = fb.create_dataset("f64_2d"); + ds.with_f64_data(&ramp_f64(37 * 53)) + .with_shape(&[37, 53]) + .with_chunks(&[10, 16]); + configure(ds); + } + { + let ds = fb.create_dataset("u8_1d"); + ds.with_u8_data(&ramp_u8(5000)).with_chunks(&[777]); + configure(ds); + } + { + let ds = fb.create_dataset("f64_big"); + ds.with_f64_data(&ramp_f64(100_000)).with_chunks(&[40_000]); + configure(ds); + } + fb.write(&path).unwrap(); + + // clawhdf5 reads its own output. + let file = File::open(&path).unwrap(); + assert_eq!( + file.dataset("i32_1d").unwrap().read_i32().unwrap(), + ramp_i32(10_000) + ); + assert_eq!( + file.dataset("f64_2d").unwrap().read_f64().unwrap(), + ramp_f64(37 * 53) + ); + + let out = run_python( + VERIFY, + &[path.to_str().unwrap(), &filter_id.to_string(), filter_name], + ); + assert_eq!(out, "OK", "{tag}: h5py could not read our output"); +} + +#[cfg(feature = "lzf")] +#[test] +fn lzf_written_by_h5py_reads_exactly() { + if !have_python("h5py") { + return; + } + check_h5py_written( + "lzf", + r#"[('lzf', dict(compression='lzf')), + ('lzf+shuffle', dict(compression='lzf', shuffle=True)), + ('lzf+shuffle+fletcher32', dict(compression='lzf', shuffle=True, fletcher32=True))]"#, + ); +} + +#[cfg(feature = "lzf")] +#[test] +fn lzf_written_by_clawhdf5_reads_in_h5py() { + if !have_python("h5py") { + return; + } + check_ours_read_by_h5py("lzf", 32000, "lzf", |ds| { + ds.with_lzf(); + }); + check_ours_read_by_h5py("lzf_noshuffle", 32000, "lzf", |ds| { + ds.with_lzf().without_shuffle(); + }); +} From 07094e34a9b0003fbad5b46744cd63d7d03436cb Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 00:04:05 -0500 Subject: [PATCH 03/13] feat(format): bitshuffle filter (32008), with its LZ4 and Zstandard modes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit hdf5plugin's Bitshuffle failed with UnsupportedFilter(32008). The new `bitshuffle` feature (pure Rust: lz4_flex, and ruzstd for Zstandard — the `zstd` feature's libzstd is not needed) decodes all three modes of bshuf_h5filter.c — transpose only, LZ4 and Zstandard blocks behind the 12-byte header — including the default and explicit block sizes, the shorter last block rounded down to a multiple of 8 elements, and the untransposed trailing elements. Sizes read from the chunk are bounded by the chunk size. It also encodes: DatasetBuilder::with_bitshuffle(BitshuffleCompression) or PluginFilter::Bitshuffle { block_size, compression } writes the filter with hdf5plugin's cd_values and no automatic byte shuffle. ruzstd has one compression level (about zstd's 1); the requested level is recorded. The bit transpose is checked bit for bit against a one-bit-at-a-time model (which matched hdf5plugin's output) and is shared with blosc next. Interop: hdf5plugin writes none/LZ4/Zstandard at default and explicit block sizes and levels over the 12-case matrix, read byte for byte; our three modes at two block sizes read back through hdf5plugin. Both fail with the decoder removed. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/Cargo.toml | 5 + crates/clawhdf5-format/src/chunked_write.rs | 65 ++- crates/clawhdf5-format/src/filters.rs | 7 + .../clawhdf5-format/src/filters_bitshuffle.rs | 377 ++++++++++++++++++ crates/clawhdf5-format/src/lib.rs | 2 + crates/clawhdf5-format/src/type_builders.rs | 14 + crates/clawhdf5/Cargo.toml | 1 + .../clawhdf5/tests/plugin_filters_interop.rs | 41 ++ 8 files changed, 509 insertions(+), 3 deletions(-) create mode 100644 crates/clawhdf5-format/src/filters_bitshuffle.rs diff --git a/crates/clawhdf5-format/Cargo.toml b/crates/clawhdf5-format/Cargo.toml index 4eb4c21..f6a9087 100644 --- a/crates/clawhdf5-format/Cargo.toml +++ b/crates/clawhdf5-format/Cargo.toml @@ -22,6 +22,9 @@ zstd = { version = "0.13", optional = true } blake3 = { version = "1", optional = true } libaec-sys = { path = "../libaec-sys", version = "0.1", optional = true } pco = { version = "1.0", optional = true } +# Pure-Rust Zstandard, for the plugin filters that embed zstd (bitshuffle, +# blosc). The `zstd` feature (filter 32015) links libzstd instead. +ruzstd = { version = "0.9", optional = true } [dev-dependencies] half = { workspace = true } @@ -59,6 +62,8 @@ pcodec = ["dep:pco"] # Plugin filters, pure Rust. LZF (32000) is h5py's built-in compression; it # has no dependencies, so it is on by default. lzf = [] +# Bitshuffle (32008), with its LZ4 and Zstandard modes. +bitshuffle = ["lz4_flex", "ruzstd"] [[bench]] name = "parallel_decompress_bench" diff --git a/crates/clawhdf5-format/src/chunked_write.rs b/crates/clawhdf5-format/src/chunked_write.rs index 4e9b851..e155c89 100644 --- a/crates/clawhdf5-format/src/chunked_write.rs +++ b/crates/clawhdf5-format/src/chunked_write.rs @@ -12,8 +12,8 @@ use crate::chunk_grid::ChunkGrid; use crate::ea_writer; use crate::error::FormatError; use crate::filter_pipeline::{ - FILTER_DEFLATE, FILTER_FLETCHER32, FILTER_LZ4, FILTER_LZF, FILTER_PCODEC, FILTER_PCODEC_NAME, - FILTER_SHUFFLE, FILTER_ZSTD, FilterDescription, FilterPipeline, + FILTER_BITSHUFFLE, FILTER_DEFLATE, FILTER_FLETCHER32, FILTER_LZ4, FILTER_LZF, FILTER_PCODEC, + FILTER_PCODEC_NAME, FILTER_SHUFFLE, FILTER_ZSTD, FilterDescription, FilterPipeline, }; use crate::filters::compress_chunk; /// Round a file offset up to the next cache-line boundary. @@ -61,6 +61,30 @@ pub enum PluginFilter { /// LZF (filter 32000), h5py's built-in `compression="lzf"`. Needs the /// `lzf` feature. Lzf, + /// Bitshuffle (filter 32008): a bit transpose of each block of + /// `block_size` elements (0 = bitshuffle's default, else a multiple of + /// 8), optionally compressed. Needs the `bitshuffle` feature. + Bitshuffle { + /// Block size in elements; 0 for the default. + block_size: u32, + /// Compression after the transpose. + compression: BitshuffleCompression, + }, +} + +/// What bitshuffle compresses its blocks with. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BitshuffleCompression { + /// Transpose only. + None, + /// LZ4 (bitshuffle's `cname="lz4"`, the common choice). + Lz4, + /// Zstandard. clawhdf5's pure-Rust encoder has a single level (about + /// zstd's level 1); `level` is recorded in the file for other writers. + Zstd { + /// Level recorded in `cd_values[5]`. + level: u32, + }, } impl PluginFilter { @@ -69,12 +93,13 @@ impl PluginFilter { fn shuffles_itself(&self) -> bool { match self { PluginFilter::Lzf => false, + PluginFilter::Bitshuffle { .. } => true, } } /// The pipeline entry for this filter. `chunk_bytes` is one chunk's /// uncompressed size (0 if unknown). - fn description(&self, _element_size: u32, chunk_bytes: u32) -> FilterDescription { + fn description(&self, element_size: u32, chunk_bytes: u32) -> FilterDescription { match self { // h5py's lzf_set_local: filter version, liblzf version, chunk // size in bytes. Optional, as h5py flags it: a chunk the filter @@ -85,6 +110,25 @@ impl PluginFilter { flags: 1, client_data: vec![4, 0x0105, chunk_bytes], }, + // bshuf_h5_set_local: version 0.4, element size, block size, + // compression (0 none, 2 LZ4, 3 Zstandard), Zstandard level. + PluginFilter::Bitshuffle { + block_size, + compression, + } => { + let mut cd = vec![0, 4, element_size, *block_size]; + match compression { + BitshuffleCompression::None => cd.push(0), + BitshuffleCompression::Lz4 => cd.push(2), + BitshuffleCompression::Zstd { level } => cd.extend([3, *level]), + } + FilterDescription { + filter_id: FILTER_BITSHUFFLE, + name: Some("bitshuffle; see https://github.com/kiyo-masui/bitshuffle".into()), + flags: 1, + client_data: cd, + } + } } } } @@ -1649,6 +1693,21 @@ mod tests { assert_eq!(pl.filters[1].client_data, vec![4, 0x0105, 800]); } + #[test] + fn chunk_options_pipeline_bitshuffle_has_no_auto_shuffle() { + let options = ChunkOptions { + plugin: Some(PluginFilter::Bitshuffle { + block_size: 0, + compression: BitshuffleCompression::Zstd { level: 5 }, + }), + ..Default::default() + }; + let pl = options.build_pipeline(4).unwrap(); + assert_eq!(pl.filters.len(), 1); + assert_eq!(pl.filters[0].filter_id, FILTER_BITSHUFFLE); + assert_eq!(pl.filters[0].client_data, vec![0, 4, 4, 0, 3, 5]); + } + #[test] fn chunk_options_zstd_priority_over_deflate() { let options = ChunkOptions { diff --git a/crates/clawhdf5-format/src/filters.rs b/crates/clawhdf5-format/src/filters.rs index 237ceb5..5a61251 100644 --- a/crates/clawhdf5-format/src/filters.rs +++ b/crates/clawhdf5-format/src/filters.rs @@ -206,6 +206,13 @@ pub(crate) static BUILTIN_FILTERS: &[BuiltinFilter] = &[ decode: |d, c| lz4_decompress(d, c.max_output), encode: Some(|d, c| lz4_compress(d, c.client_data())), }, + #[cfg(feature = "bitshuffle")] + BuiltinFilter { + id: crate::filter_pipeline::FILTER_BITSHUFFLE, + name: "bitshuffle", + decode: crate::filters_bitshuffle::bitshuffle_decode, + encode: Some(crate::filters_bitshuffle::bitshuffle_encode), + }, #[cfg(feature = "zstd")] BuiltinFilter { id: FILTER_ZSTD, diff --git a/crates/clawhdf5-format/src/filters_bitshuffle.rs b/crates/clawhdf5-format/src/filters_bitshuffle.rs new file mode 100644 index 0000000..9844adc --- /dev/null +++ b/crates/clawhdf5-format/src/filters_bitshuffle.rs @@ -0,0 +1,377 @@ +//! Bitshuffle (HDF5 filter 32008) and the bit transpose it shares with blosc. +//! +//! **The transform.** A block of `n` elements (`n` a multiple of 8) of +//! `es` bytes each is viewed as an `n × 8·es` bit matrix — row *i* is +//! element *i*, column `8·j + k` is bit *k* (LSB first) of its byte *j* — and +//! transposed: the output is `8·es` rows of `n` bits, row `8·j + k` holding +//! bit *k* of byte *j* of every element in order, packed LSB first. That is +//! what `bshuf_trans_bit_elem` produces (checked against hdf5plugin's +//! library bit for bit). +//! +//! **The filter** (`bshuf_h5filter.c`). `cd_values`: `[0..2]` bitshuffle +//! version, `[2]` element size, `[3]` block size in elements (0 = default: +//! 8192 bytes' worth, rounded down to a multiple of 8, at least 128), +//! `[4]` compression (0 none, 2 LZ4, 3 Zstandard), `[5]` Zstandard level. +//! The chunk is cut into blocks of `block size` elements; the tail shorter +//! than a block is transposed as one block rounded down to a multiple of 8 +//! elements, and the last `n mod 8` elements are stored as they are. +//! Uncompressed, that is the whole chunk. Compressed, the chunk starts with a +//! 12-byte header — the decoded size (u64 big-endian) and the block size in +//! bytes (u32 big-endian) — and each transposed block is stored as a u32 +//! big-endian length and an LZ4 block / Zstandard frame; the untransposed +//! tail follows the last block. + +#[cfg(not(feature = "std"))] +extern crate alloc; +#[cfg(not(feature = "std"))] +use alloc::{format, vec, vec::Vec}; + +use crate::error::FormatError; +#[cfg(feature = "bitshuffle")] +use crate::filter_registry::FilterContext; + +/// Transpose an 8×8 bit matrix packed in a u64 (byte *r* = row *r*, bit *c* +/// of that byte = column *c*). An involution. +#[inline] +fn transpose8(mut x: u64) -> u64 { + let t = (x ^ (x >> 7)) & 0x00AA_00AA_00AA_00AA; + x = x ^ t ^ (t << 7); + let t = (x ^ (x >> 14)) & 0x0000_CCCC_0000_CCCC; + x = x ^ t ^ (t << 14); + let t = (x ^ (x >> 28)) & 0x0000_0000_F0F0_F0F0; + x ^ t ^ (t << 28) +} + +/// Bit-transpose one block: `input` and `out` are `n * es` bytes, `n` a +/// multiple of 8. +pub(crate) fn bitshuffle_block(input: &[u8], out: &mut [u8], n: usize, es: usize) { + debug_assert!(n.is_multiple_of(8) && input.len() == n * es && out.len() == n * es); + let row = n / 8; + for j in 0..es { + for g in 0..row { + let mut x = 0u64; + for t in 0..8 { + x |= u64::from(input[(8 * g + t) * es + j]) << (8 * t); + } + let y = transpose8(x); + for k in 0..8 { + out[(8 * j + k) * row + g] = (y >> (8 * k)) as u8; + } + } + } +} + +/// Undo [`bitshuffle_block`]. +pub(crate) fn bitunshuffle_block(input: &[u8], out: &mut [u8], n: usize, es: usize) { + debug_assert!(n.is_multiple_of(8) && input.len() == n * es && out.len() == n * es); + let row = n / 8; + for j in 0..es { + for g in 0..row { + let mut y = 0u64; + for k in 0..8 { + y |= u64::from(input[(8 * j + k) * row + g]) << (8 * k); + } + let x = transpose8(y); + for t in 0..8 { + out[(8 * g + t) * es + j] = (x >> (8 * t)) as u8; + } + } + } +} + +/// `bshuf_default_block_size`: 8 KiB of elements, a multiple of 8, >= 128. +#[cfg(feature = "bitshuffle")] +fn default_block_size(es: usize) -> usize { + ((8192 / es) / 8 * 8).max(128) +} + +#[cfg(feature = "bitshuffle")] +fn err(msg: &str) -> FormatError { + FormatError::DecompressionError(format!("bitshuffle: {msg}")) +} + +/// `cd_values[4]`: the compression bitshuffle applies after the transpose. +#[cfg(feature = "bitshuffle")] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Codec { + None, + Lz4, + Zstd, +} + +#[cfg(feature = "bitshuffle")] +fn codec(cd: &[u32]) -> Result { + match cd.get(4).copied().unwrap_or(0) { + 0 => Ok(Codec::None), + 2 => Ok(Codec::Lz4), + 3 => Ok(Codec::Zstd), + other => Err(FormatError::FilterError(format!( + "bitshuffle: unknown compression {other}" + ))), + } +} + +/// The element counts of the transposed blocks for `size` elements. +#[cfg(feature = "bitshuffle")] +fn blocks(size: usize, block: usize) -> impl Iterator { + let full = size / block; + let last = (size % block) / 8 * 8; + core::iter::repeat_n(block, full).chain((last > 0).then_some(last)) +} + +/// Decode a bitshuffle-filtered chunk. +#[cfg(feature = "bitshuffle")] +pub(crate) fn bitshuffle_decode( + input: &[u8], + ctx: &FilterContext<'_>, +) -> Result, FormatError> { + let cd = ctx.client_data(); + let es = match cd.get(2) { + Some(&e) if e != 0 => e as usize, + _ => return Err(err("missing element size")), + }; + let codec = codec(cd)?; + let limit = ctx.output_limit(); + if codec == Codec::None { + if input.len() > limit { + return Err(err("output exceeds the chunk size")); + } + let block = match cd.get(3) { + Some(&b) if b != 0 => b as usize, + _ => default_block_size(es), + }; + if !block.is_multiple_of(8) { + return Err(err("block size is not a multiple of 8")); + } + if !input.len().is_multiple_of(es) { + return Err(err("chunk is not a whole number of elements")); + } + let size = input.len() / es; + let mut out = vec![0u8; input.len()]; + let mut pos = 0; + for n in blocks(size, block) { + let bytes = n * es; + bitunshuffle_block(&input[pos..pos + bytes], &mut out[pos..pos + bytes], n, es); + pos += bytes; + } + out[pos..].copy_from_slice(&input[pos..]); + return Ok(out); + } + + let header = input.get(..12).ok_or_else(|| err("truncated header"))?; + let total = u64::from_be_bytes(header[..8].try_into().unwrap()); + let block_bytes = u32::from_be_bytes(header[8..12].try_into().unwrap()) as usize; + let total = usize::try_from(total) + .ok() + .filter(|&t| t <= limit) + .ok_or_else(|| err("decoded size exceeds the chunk size"))?; + if !total.is_multiple_of(es) { + return Err(err("chunk is not a whole number of elements")); + } + if block_bytes == 0 || !block_bytes.is_multiple_of(es) { + return Err(err("bad block size")); + } + let block = block_bytes / es; + if !block.is_multiple_of(8) { + return Err(err("block size is not a multiple of 8")); + } + let size = total / es; + let mut out = vec![0u8; total]; + let mut tmp = vec![0u8; block_bytes.min(total)]; + let mut ip = 12usize; + let mut op = 0usize; + let mut zstd = None; + for n in blocks(size, block) { + let bytes = n * es; + let len = input + .get(ip..ip + 4) + .map(|b| u32::from_be_bytes(b.try_into().unwrap()) as usize) + .ok_or_else(|| err("truncated block header"))?; + ip += 4; + let comp = input + .get(ip..ip.saturating_add(len)) + .ok_or_else(|| err("truncated block"))?; + ip += len; + let dst = &mut tmp[..bytes]; + let got = match codec { + Codec::Lz4 => lz4_flex::block::decompress_into(comp, dst) + .map_err(|e| err(&format!("lz4: {e}")))?, + Codec::Zstd => zstd_decode_into( + zstd.get_or_insert_with(ruzstd::decoding::FrameDecoder::new), + comp, + dst, + )?, + Codec::None => unreachable!(), + }; + if got != bytes { + return Err(err("block decoded to the wrong size")); + } + bitunshuffle_block(dst, &mut out[op..op + bytes], n, es); + op += bytes; + } + let tail = total - op; + let rest = input + .get(ip..ip + tail) + .ok_or_else(|| err("truncated trailing elements"))?; + out[op..].copy_from_slice(rest); + Ok(out) +} + +/// Decode Zstandard frames into exactly `dst`, failing if they hold more. +#[cfg(feature = "bitshuffle")] +pub(crate) fn zstd_decode_into( + decoder: &mut ruzstd::decoding::FrameDecoder, + frames: &[u8], + dst: &mut [u8], +) -> Result { + decoder + .decode_all(frames, dst) + .map_err(|e| FormatError::DecompressionError(format!("zstd: {e}"))) +} + +/// Compress with ruzstd. It implements one level (roughly zstd's level 1), +/// so the requested level only matters to other encoders. +#[cfg(feature = "bitshuffle")] +pub(crate) fn zstd_encode(data: &[u8]) -> Vec { + ruzstd::encoding::compress_to_vec(data, ruzstd::encoding::CompressionLevel::Fastest) +} + +/// Encode a chunk with the bitshuffle filter. +#[cfg(feature = "bitshuffle")] +pub(crate) fn bitshuffle_encode( + input: &[u8], + ctx: &FilterContext<'_>, +) -> Result, FormatError> { + let cd = ctx.client_data(); + let es = match cd.get(2) { + Some(&e) if e != 0 => e as usize, + _ => ctx.element_size.max(1), + }; + let codec = codec(cd)?; + let block = match cd.get(3) { + Some(&b) if b != 0 => b as usize, + _ => default_block_size(es), + }; + let cerr = |m: &str| FormatError::CompressionError(format!("bitshuffle: {m}")); + if !block.is_multiple_of(8) { + return Err(cerr("block size is not a multiple of 8")); + } + if !input.len().is_multiple_of(es) { + return Err(cerr("chunk is not a whole number of elements")); + } + let size = input.len() / es; + let mut out = Vec::with_capacity(input.len() + 12 + input.len() / 64); + if codec != Codec::None { + out.extend_from_slice(&(input.len() as u64).to_be_bytes()); + let block_bytes = + u32::try_from(block * es).map_err(|_| cerr("block size does not fit in 32 bits"))?; + out.extend_from_slice(&block_bytes.to_be_bytes()); + } + let mut tmp = vec![0u8; (block * es).min(input.len())]; + let mut pos = 0; + for n in blocks(size, block) { + let bytes = n * es; + let dst = &mut tmp[..bytes]; + bitshuffle_block(&input[pos..pos + bytes], dst, n, es); + match codec { + Codec::None => out.extend_from_slice(dst), + Codec::Lz4 | Codec::Zstd => { + let comp = if codec == Codec::Lz4 { + lz4_flex::block::compress(dst) + } else { + zstd_encode(dst) + }; + out.extend_from_slice(&(comp.len() as u32).to_be_bytes()); + out.extend_from_slice(&comp); + } + } + pos += bytes; + } + out.extend_from_slice(&input[pos..]); + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The definition, one bit at a time. + fn naive(input: &[u8], n: usize, es: usize) -> Vec { + let mut out = vec![0u8; n * es]; + for i in 0..n { + for j in 0..es { + for k in 0..8 { + if input[i * es + j] >> k & 1 == 1 { + let p = (8 * j + k) * n + i; + out[p / 8] |= 1 << (p % 8); + } + } + } + } + out + } + + #[test] + fn transpose_matches_the_definition_and_inverts() { + for (n, es) in [(8, 1), (16, 2), (24, 4), (128, 8), (64, 3), (8, 16)] { + let input: Vec = (0..n * es) + .map(|i| (i as u32).wrapping_mul(2_654_435_761).rotate_left(7) as u8) + .collect(); + let mut out = vec![0u8; n * es]; + bitshuffle_block(&input, &mut out, n, es); + assert_eq!(out, naive(&input, n, es), "n={n} es={es}"); + let mut back = vec![0u8; n * es]; + bitunshuffle_block(&out, &mut back, n, es); + assert_eq!(back, input); + } + } + + #[cfg(feature = "bitshuffle")] + fn ctx_for(cd: Vec) -> crate::filter_pipeline::FilterDescription { + crate::filter_pipeline::FilterDescription { + filter_id: crate::filter_pipeline::FILTER_BITSHUFFLE, + name: None, + flags: 0, + client_data: cd, + } + } + + #[cfg(feature = "bitshuffle")] + #[test] + fn filter_round_trips_every_mode() { + for es in [1usize, 2, 4, 8] { + for n in [0usize, 1, 7, 8, 100, 1000, 5003] { + let data: Vec = (0..n * es) + .map(|i| (i % 97) as u8 ^ (i / 300) as u8) + .collect(); + for (comp, block) in [(0, 0), (0, 16), (2, 0), (2, 64), (3, 0), (3, 1024)] { + let f = ctx_for(vec![0, 4, es as u32, block, comp]); + let ctx = FilterContext { + filter: &f, + element_size: es, + max_output: data.len(), + }; + let enc = bitshuffle_encode(&data, &ctx).unwrap(); + let dec = bitshuffle_decode(&enc, &ctx).unwrap(); + assert_eq!(dec, data, "es={es} n={n} comp={comp} block={block}"); + } + } + } + } + + #[cfg(feature = "bitshuffle")] + #[test] + fn rejects_oversized_and_truncated_chunks() { + let data = vec![5u8; 4096]; + let f = ctx_for(vec![0, 4, 4, 0, 2]); + let mut ctx = FilterContext { + filter: &f, + element_size: 4, + max_output: data.len(), + }; + let enc = bitshuffle_encode(&data, &ctx).unwrap(); + assert!(bitshuffle_decode(&enc[..enc.len() - 1], &ctx).is_err()); + ctx.max_output = 100; + assert!(bitshuffle_decode(&enc, &ctx).is_err()); + } +} diff --git a/crates/clawhdf5-format/src/lib.rs b/crates/clawhdf5-format/src/lib.rs index c717d89..e043567 100644 --- a/crates/clawhdf5-format/src/lib.rs +++ b/crates/clawhdf5-format/src/lib.rs @@ -73,6 +73,8 @@ pub mod fill_value; pub mod filter_pipeline; pub mod filter_registry; pub mod filters; +#[cfg(feature = "bitshuffle")] +mod filters_bitshuffle; #[cfg(feature = "lzf")] pub mod filters_lzf; mod filters_szip; diff --git a/crates/clawhdf5-format/src/type_builders.rs b/crates/clawhdf5-format/src/type_builders.rs index c49ccc7..d88063c 100644 --- a/crates/clawhdf5-format/src/type_builders.rs +++ b/crates/clawhdf5-format/src/type_builders.rs @@ -749,6 +749,20 @@ impl DatasetBuilder { self.with_plugin_filter(crate::chunked_write::PluginFilter::Lzf) } + /// Enable bitshuffle (filter 32008) with `compression` after the bit + /// transpose, in bitshuffle's default block size. Implies chunked + /// storage; no byte shuffle is added. Requires the `bitshuffle` cargo + /// feature. + pub fn with_bitshuffle( + &mut self, + compression: crate::chunked_write::BitshuffleCompression, + ) -> &mut Self { + self.with_plugin_filter(crate::chunked_write::PluginFilter::Bitshuffle { + block_size: 0, + compression, + }) + } + /// Enable Pcodec lossless numerical compression (private clawhdf5 filter /// ID 480). /// diff --git a/crates/clawhdf5/Cargo.toml b/crates/clawhdf5/Cargo.toml index f4c6ee2..52b8ed3 100644 --- a/crates/clawhdf5/Cargo.toml +++ b/crates/clawhdf5/Cargo.toml @@ -44,6 +44,7 @@ pcodec = ["clawhdf5-format/pcodec"] # Plugin filters, pure Rust (no C). LZF (32000) is h5py's built-in # compression; it has no dependencies, so it is on by default. lzf = ["clawhdf5-format/lzf"] +bitshuffle = ["clawhdf5-format/bitshuffle"] # Dataset::verify_provenance() — recompute a dataset's SHA-256 and compare # against its stored _provenance_sha256 attribute. On by default, matching # clawhdf5-format's own default-on `provenance` feature. diff --git a/crates/clawhdf5/tests/plugin_filters_interop.rs b/crates/clawhdf5/tests/plugin_filters_interop.rs index 041aff1..f8e44a1 100644 --- a/crates/clawhdf5/tests/plugin_filters_interop.rs +++ b/crates/clawhdf5/tests/plugin_filters_interop.rs @@ -234,6 +234,47 @@ fn check_ours_read_by_h5py( assert_eq!(out, "OK", "{tag}: h5py could not read our output"); } +#[cfg(feature = "bitshuffle")] +#[test] +fn bitshuffle_written_by_hdf5plugin_reads_exactly() { + if !have_python("h5py, hdf5plugin") { + return; + } + check_h5py_written( + "bitshuffle", + r#"[('none', hdf5plugin.Bitshuffle(cname='none')), + ('lz4', hdf5plugin.Bitshuffle(cname='lz4')), + ('lz4 nelems=16', hdf5plugin.Bitshuffle(nelems=16, cname='lz4')), + ('none nelems=64', hdf5plugin.Bitshuffle(nelems=64, cname='none')), + ('zstd', hdf5plugin.Bitshuffle(cname='zstd')), + ('zstd clevel=19 nelems=2048', hdf5plugin.Bitshuffle(nelems=2048, cname='zstd', clevel=19))]"#, + ); +} + +#[cfg(feature = "bitshuffle")] +#[test] +fn bitshuffle_written_by_clawhdf5_reads_in_hdf5plugin() { + use clawhdf5_format::chunked_write::{BitshuffleCompression, PluginFilter}; + if !have_python("h5py, hdf5plugin") { + return; + } + for (tag, compression) in [ + ("bshuf_none", BitshuffleCompression::None), + ("bshuf_lz4", BitshuffleCompression::Lz4), + ("bshuf_zstd", BitshuffleCompression::Zstd { level: 3 }), + ] { + check_ours_read_by_h5py(tag, 32008, "bitshuffle", |ds| { + ds.with_bitshuffle(compression); + }); + check_ours_read_by_h5py(tag, 32008, "bitshuffle", |ds| { + ds.with_plugin_filter(PluginFilter::Bitshuffle { + block_size: 40, + compression, + }); + }); + } +} + #[cfg(feature = "lzf")] #[test] fn lzf_written_by_h5py_reads_exactly() { From 6dfd2390112d132a517388126407a0d3610290ed Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 00:05:26 -0500 Subject: [PATCH 04/13] feat(format): bzip2 filter (307), read and write, pure Rust hdf5plugin's BZip2 failed with UnsupportedFilter(307). The new `bzip2` feature decodes the single bzip2 stream H5Zbzip2.c stores, bounded by the chunk size (a truncated stream is an error, not short data), and encodes at block size cd_values[0] (DatasetBuilder::with_bzip2(level)). It uses the bzip2 crate's default backend, libbz2-rs-sys, a pure-Rust port of libbzip2: `cargo tree` shows no cc/cmake, and nothing is compiled from C. Interop: hdf5plugin writes block sizes 9, 1 and 5+shuffle over the 12-case matrix, read byte for byte; ours at 9 (shuffled) and 1 (not) reads back through hdf5plugin. Both fail with the decoder removed. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/Cargo.toml | 5 + crates/clawhdf5-format/src/chunked_write.rs | 18 ++- crates/clawhdf5-format/src/filters.rs | 7 ++ crates/clawhdf5-format/src/filters_bzip2.rs | 112 ++++++++++++++++++ crates/clawhdf5-format/src/lib.rs | 2 + crates/clawhdf5-format/src/type_builders.rs | 7 ++ crates/clawhdf5/Cargo.toml | 1 + .../clawhdf5/tests/plugin_filters_interop.rs | 28 +++++ 8 files changed, 178 insertions(+), 2 deletions(-) create mode 100644 crates/clawhdf5-format/src/filters_bzip2.rs diff --git a/crates/clawhdf5-format/Cargo.toml b/crates/clawhdf5-format/Cargo.toml index f6a9087..007d8e7 100644 --- a/crates/clawhdf5-format/Cargo.toml +++ b/crates/clawhdf5-format/Cargo.toml @@ -25,6 +25,9 @@ pco = { version = "1.0", optional = true } # Pure-Rust Zstandard, for the plugin filters that embed zstd (bitshuffle, # blosc). The `zstd` feature (filter 32015) links libzstd instead. ruzstd = { version = "0.9", optional = true } +# bzip2 with its default backend, libbz2-rs-sys: a pure-Rust port of +# libbzip2 (no C is compiled, despite the -sys name). +bzip2 = { version = "0.6", optional = true } [dev-dependencies] half = { workspace = true } @@ -64,6 +67,8 @@ pcodec = ["dep:pco"] lzf = [] # Bitshuffle (32008), with its LZ4 and Zstandard modes. bitshuffle = ["lz4_flex", "ruzstd"] +# bzip2 (307). +bzip2 = ["dep:bzip2", "std"] [[bench]] name = "parallel_decompress_bench" diff --git a/crates/clawhdf5-format/src/chunked_write.rs b/crates/clawhdf5-format/src/chunked_write.rs index e155c89..e825aed 100644 --- a/crates/clawhdf5-format/src/chunked_write.rs +++ b/crates/clawhdf5-format/src/chunked_write.rs @@ -12,8 +12,9 @@ use crate::chunk_grid::ChunkGrid; use crate::ea_writer; use crate::error::FormatError; use crate::filter_pipeline::{ - FILTER_BITSHUFFLE, FILTER_DEFLATE, FILTER_FLETCHER32, FILTER_LZ4, FILTER_LZF, FILTER_PCODEC, - FILTER_PCODEC_NAME, FILTER_SHUFFLE, FILTER_ZSTD, FilterDescription, FilterPipeline, + FILTER_BITSHUFFLE, FILTER_BZIP2, FILTER_DEFLATE, FILTER_FLETCHER32, FILTER_LZ4, FILTER_LZF, + FILTER_PCODEC, FILTER_PCODEC_NAME, FILTER_SHUFFLE, FILTER_ZSTD, FilterDescription, + FilterPipeline, }; use crate::filters::compress_chunk; /// Round a file offset up to the next cache-line boundary. @@ -70,6 +71,12 @@ pub enum PluginFilter { /// Compression after the transpose. compression: BitshuffleCompression, }, + /// bzip2 (filter 307) at block size `level` (1-9). Needs the `bzip2` + /// feature. + Bzip2 { + /// Block size 1-9 (9 = hdf5plugin's default). + level: u32, + }, } /// What bitshuffle compresses its blocks with. @@ -94,6 +101,7 @@ impl PluginFilter { match self { PluginFilter::Lzf => false, PluginFilter::Bitshuffle { .. } => true, + PluginFilter::Bzip2 { .. } => false, } } @@ -112,6 +120,12 @@ impl PluginFilter { }, // bshuf_h5_set_local: version 0.4, element size, block size, // compression (0 none, 2 LZ4, 3 Zstandard), Zstandard level. + PluginFilter::Bzip2 { level } => FilterDescription { + filter_id: FILTER_BZIP2, + name: Some("bzip2".into()), + flags: 1, + client_data: vec![(*level).clamp(1, 9)], + }, PluginFilter::Bitshuffle { block_size, compression, diff --git a/crates/clawhdf5-format/src/filters.rs b/crates/clawhdf5-format/src/filters.rs index 5a61251..d744cd0 100644 --- a/crates/clawhdf5-format/src/filters.rs +++ b/crates/clawhdf5-format/src/filters.rs @@ -185,6 +185,13 @@ pub(crate) static BUILTIN_FILTERS: &[BuiltinFilter] = &[ decode: |d, c| scaleoffset_decompress(d, c.client_data(), c.max_output), encode: None, }, + #[cfg(feature = "bzip2")] + BuiltinFilter { + id: crate::filter_pipeline::FILTER_BZIP2, + name: "bzip2", + decode: crate::filters_bzip2::bzip2_decode, + encode: Some(crate::filters_bzip2::bzip2_encode), + }, #[cfg(feature = "pcodec")] BuiltinFilter { id: FILTER_PCODEC, diff --git a/crates/clawhdf5-format/src/filters_bzip2.rs b/crates/clawhdf5-format/src/filters_bzip2.rs new file mode 100644 index 0000000..2a78074 --- /dev/null +++ b/crates/clawhdf5-format/src/filters_bzip2.rs @@ -0,0 +1,112 @@ +//! bzip2 (HDF5 filter 307, PyTables' `H5Zbzip2.c`, hdf5plugin's `BZip2`). +//! +//! The chunk is one bzip2 stream; `cd_values[0]` is the block size (1-9, +//! the compression level). Decoded with the `bzip2` crate's default backend, +//! `libbz2-rs-sys`, a pure-Rust port of libbzip2. + +use crate::error::FormatError; +use crate::filter_registry::FilterContext; + +fn err(msg: &str) -> FormatError { + FormatError::DecompressionError(format!("bzip2: {msg}")) +} + +/// Decode a bzip2-filtered chunk, refusing output beyond the chunk size. +pub(crate) fn bzip2_decode(input: &[u8], ctx: &FilterContext<'_>) -> Result, FormatError> { + use bzip2::{Decompress, Status}; + let limit = ctx.output_limit(); + let max_capacity = limit.saturating_add(1); + let hint = if ctx.max_output != 0 { + ctx.max_output + } else { + input.len().saturating_mul(4) + }; + let mut out = Vec::new(); + out.try_reserve_exact(hint.clamp(1, max_capacity)) + .map_err(|_| err("cannot allocate the output buffer"))?; + let mut dec = Decompress::new(false); + loop { + let (in_before, out_before) = (dec.total_in(), dec.total_out()); + let status = dec + .decompress_vec(&input[in_before as usize..], &mut out) + .map_err(|e| err(&e.to_string()))?; + if out.len() > limit { + return Err(err("output exceeds the chunk size")); + } + if status == Status::StreamEnd { + return Ok(out); + } + if out.len() == out.capacity() { + let grow = out.capacity().min(max_capacity - out.capacity()).max(1); + out.try_reserve_exact(grow) + .map_err(|_| err("cannot allocate the output buffer"))?; + } else if dec.total_in() as usize >= input.len() + || (dec.total_in(), dec.total_out()) == (in_before, out_before) + { + return Err(err("truncated stream")); + } + } +} + +/// Encode a chunk as one bzip2 stream at block size `cd_values[0]` +/// (default 9, as hdf5plugin). +pub(crate) fn bzip2_encode(input: &[u8], ctx: &FilterContext<'_>) -> Result, FormatError> { + use bzip2::{Action, Compress, Compression, Status}; + let level = ctx.client_data().first().copied().unwrap_or(9).clamp(1, 9); + let cerr = |m: String| FormatError::CompressionError(format!("bzip2: {m}")); + let mut enc = Compress::new(Compression::new(level), 0); + // bzip2's worst case is about 1% + 600 bytes over the input. + let mut out = Vec::with_capacity(input.len() + input.len() / 100 + 600); + loop { + let consumed = enc.total_in() as usize; + let status = enc + .compress_vec(&input[consumed..], &mut out, Action::Finish) + .map_err(|e| cerr(e.to_string()))?; + if status == Status::StreamEnd { + return Ok(out); + } + if out.len() == out.capacity() { + out.reserve(out.capacity().max(4096)); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::filter_pipeline::{FILTER_BZIP2, FilterDescription}; + + fn desc(level: u32) -> FilterDescription { + FilterDescription { + filter_id: FILTER_BZIP2, + name: None, + flags: 0, + client_data: vec![level], + } + } + + #[test] + fn round_trips_and_bounds() { + let data: Vec = (0..100_000u32) + .flat_map(|i| (i % 777).to_le_bytes()) + .collect(); + for level in [1, 5, 9] { + let f = desc(level); + let ctx = FilterContext { + filter: &f, + element_size: 4, + max_output: data.len(), + }; + let enc = bzip2_encode(&data, &ctx).unwrap(); + assert!(enc.len() < data.len() / 4); + assert_eq!(bzip2_decode(&enc, &ctx).unwrap(), data); + // Truncated, and larger than the chunk: errors, not data. + assert!(bzip2_decode(&enc[..enc.len() / 2], &ctx).is_err()); + let small = FilterContext { + max_output: data.len() - 1, + ..ctx + }; + assert!(bzip2_decode(&enc, &small).is_err()); + } + } +} diff --git a/crates/clawhdf5-format/src/lib.rs b/crates/clawhdf5-format/src/lib.rs index e043567..b27ca09 100644 --- a/crates/clawhdf5-format/src/lib.rs +++ b/crates/clawhdf5-format/src/lib.rs @@ -75,6 +75,8 @@ pub mod filter_registry; pub mod filters; #[cfg(feature = "bitshuffle")] mod filters_bitshuffle; +#[cfg(feature = "bzip2")] +mod filters_bzip2; #[cfg(feature = "lzf")] pub mod filters_lzf; mod filters_szip; diff --git a/crates/clawhdf5-format/src/type_builders.rs b/crates/clawhdf5-format/src/type_builders.rs index d88063c..635d66a 100644 --- a/crates/clawhdf5-format/src/type_builders.rs +++ b/crates/clawhdf5-format/src/type_builders.rs @@ -763,6 +763,13 @@ impl DatasetBuilder { }) } + /// Enable bzip2 (filter 307) at block size `level` (1-9). Implies + /// chunked storage; shuffle is applied first unless + /// `.without_shuffle()`. Requires the `bzip2` cargo feature. + pub fn with_bzip2(&mut self, level: u32) -> &mut Self { + self.with_plugin_filter(crate::chunked_write::PluginFilter::Bzip2 { level }) + } + /// Enable Pcodec lossless numerical compression (private clawhdf5 filter /// ID 480). /// diff --git a/crates/clawhdf5/Cargo.toml b/crates/clawhdf5/Cargo.toml index 52b8ed3..1523920 100644 --- a/crates/clawhdf5/Cargo.toml +++ b/crates/clawhdf5/Cargo.toml @@ -45,6 +45,7 @@ pcodec = ["clawhdf5-format/pcodec"] # compression; it has no dependencies, so it is on by default. lzf = ["clawhdf5-format/lzf"] bitshuffle = ["clawhdf5-format/bitshuffle"] +bzip2 = ["clawhdf5-format/bzip2"] # Dataset::verify_provenance() — recompute a dataset's SHA-256 and compare # against its stored _provenance_sha256 attribute. On by default, matching # clawhdf5-format's own default-on `provenance` feature. diff --git a/crates/clawhdf5/tests/plugin_filters_interop.rs b/crates/clawhdf5/tests/plugin_filters_interop.rs index f8e44a1..5cd1188 100644 --- a/crates/clawhdf5/tests/plugin_filters_interop.rs +++ b/crates/clawhdf5/tests/plugin_filters_interop.rs @@ -275,6 +275,34 @@ fn bitshuffle_written_by_clawhdf5_reads_in_hdf5plugin() { } } +#[cfg(feature = "bzip2")] +#[test] +fn bzip2_written_by_hdf5plugin_reads_exactly() { + if !have_python("h5py, hdf5plugin") { + return; + } + check_h5py_written( + "bzip2", + r#"[('bzip2 9', hdf5plugin.BZip2()), + ('bzip2 1', hdf5plugin.BZip2(blocksize=1)), + ('bzip2 5 + shuffle', dict(**hdf5plugin.BZip2(blocksize=5), shuffle=True))]"#, + ); +} + +#[cfg(feature = "bzip2")] +#[test] +fn bzip2_written_by_clawhdf5_reads_in_hdf5plugin() { + if !have_python("h5py, hdf5plugin") { + return; + } + check_ours_read_by_h5py("bzip2", 307, "bzip2", |ds| { + ds.with_bzip2(9); + }); + check_ours_read_by_h5py("bzip2_1", 307, "bzip2", |ds| { + ds.with_bzip2(1).without_shuffle(); + }); +} + #[cfg(feature = "lzf")] #[test] fn lzf_written_by_h5py_reads_exactly() { From 1f71f3bcbcc41e068fae200d586f945ced69a9a1 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 00:10:05 -0500 Subject: [PATCH 05/13] feat(format): Blosc filter (32001), read and write, pure Rust hdf5plugin's Blosc (hdf5-blosc) failed with UnsupportedFilter(32001). The new `blosc` feature decodes the Blosc 1 frame c-blosc 1.x writes: the 16-byte header, raw ("memcpyed") frames, the block table, blocks split into one stream per byte plane (and the "do not split" flag), streams stored raw, the byte shuffle and bit shuffle (whole 8-element groups, the rest copied) - and every codec hdf5plugin offers: BloscLZ (implemented here from c-blosc 1.21's blosclz_decompress, including its rejection of malformed and truncated streams), LZ4/LZ4HC (lz4_flex), Snappy (snap), Zlib (flate2) and Zstandard (ruzstd). Every stream must decode to exactly its size and the frame to at most the chunk size; a frame of another format version (Blosc 2) is a clear error. It also encodes (DatasetBuilder::with_blosc(codec, level, shuffle)): LZ4, Snappy, Zlib or Zstandard, with c-blosc's split rule, raw streams where compression does not pay, and a stored frame for level 0 or incompressible data. It cannot write BloscLZ (asking for it is an error). `plugin-filters` enables LZF, bitshuffle, bzip2 and Blosc. Interop: hdf5plugin writes all six codecs x {no, byte, bit} shuffle at levels 5/9/1, plus level 0, over the 12-case matrix, read byte for byte; our four codecs x four shuffle/level settings read back through hdf5plugin. Both fail with the decoder removed. `cargo tree` with `plugin-filters` has no -sys crate other than libbz2-rs-sys (pure Rust), no cc and no cmake. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/Cargo.toml | 5 + crates/clawhdf5-format/src/chunked_write.rs | 69 +- crates/clawhdf5-format/src/filters.rs | 7 + .../clawhdf5-format/src/filters_bitshuffle.rs | 4 +- crates/clawhdf5-format/src/filters_blosc.rs | 604 ++++++++++++++++++ crates/clawhdf5-format/src/lib.rs | 4 +- crates/clawhdf5-format/src/type_builders.rs | 16 + crates/clawhdf5/Cargo.toml | 3 + .../clawhdf5/tests/plugin_filters_interop.rs | 52 +- 9 files changed, 758 insertions(+), 6 deletions(-) create mode 100644 crates/clawhdf5-format/src/filters_blosc.rs diff --git a/crates/clawhdf5-format/Cargo.toml b/crates/clawhdf5-format/Cargo.toml index 007d8e7..75fd233 100644 --- a/crates/clawhdf5-format/Cargo.toml +++ b/crates/clawhdf5-format/Cargo.toml @@ -28,6 +28,7 @@ ruzstd = { version = "0.9", optional = true } # bzip2 with its default backend, libbz2-rs-sys: a pure-Rust port of # libbzip2 (no C is compiled, despite the -sys name). bzip2 = { version = "0.6", optional = true } +snap = { version = "1", optional = true } [dev-dependencies] half = { workspace = true } @@ -69,6 +70,10 @@ lzf = [] bitshuffle = ["lz4_flex", "ruzstd"] # bzip2 (307). bzip2 = ["dep:bzip2", "std"] +# Blosc 1 (32001) with its BloscLZ, LZ4, Snappy, Zlib and Zstandard codecs. +blosc = ["lz4_flex", "ruzstd", "snap", "deflate", "std"] +# Every plugin filter above. +plugin-filters = ["lzf", "bitshuffle", "bzip2", "blosc"] [[bench]] name = "parallel_decompress_bench" diff --git a/crates/clawhdf5-format/src/chunked_write.rs b/crates/clawhdf5-format/src/chunked_write.rs index e825aed..fef4b4f 100644 --- a/crates/clawhdf5-format/src/chunked_write.rs +++ b/crates/clawhdf5-format/src/chunked_write.rs @@ -12,8 +12,8 @@ use crate::chunk_grid::ChunkGrid; use crate::ea_writer; use crate::error::FormatError; use crate::filter_pipeline::{ - FILTER_BITSHUFFLE, FILTER_BZIP2, FILTER_DEFLATE, FILTER_FLETCHER32, FILTER_LZ4, FILTER_LZF, - FILTER_PCODEC, FILTER_PCODEC_NAME, FILTER_SHUFFLE, FILTER_ZSTD, FilterDescription, + FILTER_BITSHUFFLE, FILTER_BLOSC, FILTER_BZIP2, FILTER_DEFLATE, FILTER_FLETCHER32, FILTER_LZ4, + FILTER_LZF, FILTER_PCODEC, FILTER_PCODEC_NAME, FILTER_SHUFFLE, FILTER_ZSTD, FilterDescription, FilterPipeline, }; use crate::filters::compress_chunk; @@ -77,6 +77,41 @@ pub enum PluginFilter { /// Block size 1-9 (9 = hdf5plugin's default). level: u32, }, + /// Blosc 1 (filter 32001): `codec` at `level` (0-9; 0 stores), after + /// `shuffle`. Needs the `blosc` feature. + Blosc { + /// The codec inside the Blosc frame. + codec: BloscCodec, + /// Compression level 0-9 (0 stores the data uncompressed). + level: u32, + /// The shuffle Blosc applies first. + shuffle: BloscShuffle, + }, +} + +/// The codec inside a Blosc frame that clawhdf5 can write. (It reads +/// BloscLZ too, but cannot write it.) +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BloscCodec { + /// LZ4. + Lz4, + /// Snappy. + Snappy, + /// Zlib, at the Blosc level. + Zlib, + /// Zstandard (clawhdf5's pure-Rust encoder has one level, about zstd 1). + Zstd, +} + +/// The shuffle Blosc applies before compressing. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BloscShuffle { + /// None. + None, + /// Byte shuffle (Blosc's default). + Byte, + /// Bit shuffle. + Bit, } /// What bitshuffle compresses its blocks with. @@ -102,6 +137,7 @@ impl PluginFilter { PluginFilter::Lzf => false, PluginFilter::Bitshuffle { .. } => true, PluginFilter::Bzip2 { .. } => false, + PluginFilter::Blosc { .. } => true, } } @@ -120,6 +156,35 @@ impl PluginFilter { }, // bshuf_h5_set_local: version 0.4, element size, block size, // compression (0 none, 2 LZ4, 3 Zstandard), Zstandard level. + // hdf5-blosc's blosc_set_local: filter revision 2, Blosc format + // 2, type size, chunk size, then level, shuffle, compressor. + PluginFilter::Blosc { + codec, + level, + shuffle, + } => FilterDescription { + filter_id: FILTER_BLOSC, + name: Some("blosc".into()), + flags: 1, + client_data: vec![ + 2, + 2, + element_size, + chunk_bytes, + (*level).min(9), + match shuffle { + BloscShuffle::None => 0, + BloscShuffle::Byte => 1, + BloscShuffle::Bit => 2, + }, + match codec { + BloscCodec::Lz4 => 1, + BloscCodec::Snappy => 3, + BloscCodec::Zlib => 4, + BloscCodec::Zstd => 5, + }, + ], + }, PluginFilter::Bzip2 { level } => FilterDescription { filter_id: FILTER_BZIP2, name: Some("bzip2".into()), diff --git a/crates/clawhdf5-format/src/filters.rs b/crates/clawhdf5-format/src/filters.rs index d744cd0..aa8847c 100644 --- a/crates/clawhdf5-format/src/filters.rs +++ b/crates/clawhdf5-format/src/filters.rs @@ -206,6 +206,13 @@ pub(crate) static BUILTIN_FILTERS: &[BuiltinFilter] = &[ decode: crate::filters_lzf::lzf_decode, encode: Some(crate::filters_lzf::lzf_encode), }, + #[cfg(feature = "blosc")] + BuiltinFilter { + id: crate::filter_pipeline::FILTER_BLOSC, + name: "blosc", + decode: crate::filters_blosc::blosc_decode, + encode: Some(crate::filters_blosc::blosc_encode), + }, #[cfg(feature = "lz4")] BuiltinFilter { id: FILTER_LZ4, diff --git a/crates/clawhdf5-format/src/filters_bitshuffle.rs b/crates/clawhdf5-format/src/filters_bitshuffle.rs index 9844adc..83bab22 100644 --- a/crates/clawhdf5-format/src/filters_bitshuffle.rs +++ b/crates/clawhdf5-format/src/filters_bitshuffle.rs @@ -218,7 +218,7 @@ pub(crate) fn bitshuffle_decode( } /// Decode Zstandard frames into exactly `dst`, failing if they hold more. -#[cfg(feature = "bitshuffle")] +#[cfg(any(feature = "bitshuffle", feature = "blosc"))] pub(crate) fn zstd_decode_into( decoder: &mut ruzstd::decoding::FrameDecoder, frames: &[u8], @@ -231,7 +231,7 @@ pub(crate) fn zstd_decode_into( /// Compress with ruzstd. It implements one level (roughly zstd's level 1), /// so the requested level only matters to other encoders. -#[cfg(feature = "bitshuffle")] +#[cfg(any(feature = "bitshuffle", feature = "blosc"))] pub(crate) fn zstd_encode(data: &[u8]) -> Vec { ruzstd::encoding::compress_to_vec(data, ruzstd::encoding::CompressionLevel::Fastest) } diff --git a/crates/clawhdf5-format/src/filters_blosc.rs b/crates/clawhdf5-format/src/filters_blosc.rs new file mode 100644 index 0000000..0730c15 --- /dev/null +++ b/crates/clawhdf5-format/src/filters_blosc.rs @@ -0,0 +1,604 @@ +//! Blosc 1 (HDF5 filter 32001, `hdf5-blosc`, hdf5plugin's `Blosc`), in pure +//! Rust: the Blosc 1 frame, its byte shuffle and bit shuffle, and the +//! BloscLZ, LZ4/LZ4HC, Snappy, Zlib and Zstandard codecs inside it. +//! +//! **Frame** (c-blosc 1.x, format version 2). A 16-byte header — version +//! (2), codec format version (1), flags, type size, then little-endian `u32` +//! decoded size, block size and frame size. Flags: bit 0 byte shuffle, bit +//! 1 stored raw ("memcpyed": the data follows the header), bit 2 bit +//! shuffle, bit 4 "do not split", bits 5-7 the codec (0 BloscLZ, 1 LZ4 and +//! LZ4HC, 2 Snappy, 3 Zlib, 4 Zstandard). Unless stored raw, a table of +//! `u32` block offsets follows, one per block of `block size` bytes (the +//! last one may be shorter). A block is one stream, or — when the "do not +//! split" flag is clear, the type size is at most 16, the block holds at +//! least 128 elements, and it is not the short last block — `type size` +//! streams, one per byte plane. Each stream is a `u32` length and the +//! codec's output; a length equal to the stream's decoded size means the +//! bytes are stored raw. The decoded block is then unshuffled (byte shuffle +//! for type size > 1; bit shuffle when the block holds a multiple of 8 +//! elements, the trailing partial element copied as is). +//! +//! **Filter** (`blosc_filter.c`) `cd_values`: `[0]` filter revision, `[1]` +//! Blosc format version, `[2]` type size, `[3]` chunk size in bytes, `[4]` +//! compression level, `[5]` shuffle (0 none, 1 byte, 2 bit), `[6]` +//! compressor (0 blosclz, 1 lz4, 2 lz4hc, 3 snappy, 4 zlib, 5 zstd). The +//! decoder needs only the frame. + +use crate::error::FormatError; +use crate::filter_registry::FilterContext; +use crate::filters_bitshuffle::{bitshuffle_block, bitunshuffle_block}; + +const HEADER: usize = 16; +const FLAG_SHUFFLE: u8 = 0x01; +const FLAG_MEMCPYED: u8 = 0x02; +const FLAG_BITSHUFFLE: u8 = 0x04; +const FLAG_FUTURE: u8 = 0x08; +const FLAG_DONT_SPLIT: u8 = 0x10; +const MAX_SPLITS: usize = 16; +const MIN_BUFFERSIZE: usize = 128; + +fn err(msg: &str) -> FormatError { + FormatError::DecompressionError(format!("blosc: {msg}")) +} + +fn le32(b: &[u8], at: usize) -> Result { + b.get(at..at + 4) + .map(|s| u32::from_le_bytes(s.try_into().unwrap()) as usize) + .ok_or_else(|| err("truncated frame")) +} + +/// The codec inside a Blosc frame (flags bits 5-7). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Codec { + BloscLz, + Lz4, + Snappy, + Zlib, + Zstd, +} + +impl Codec { + fn from_flags(flags: u8) -> Result { + match flags >> 5 { + 0 => Ok(Codec::BloscLz), + 1 => Ok(Codec::Lz4), + 2 => Ok(Codec::Snappy), + 3 => Ok(Codec::Zlib), + 4 => Ok(Codec::Zstd), + other => Err(err(&format!("unknown codec {other}"))), + } + } +} + +/// Decode one codec stream into exactly `dst`. +fn decode_stream( + codec: Codec, + src: &[u8], + dst: &mut [u8], + zstd: &mut Option, +) -> Result<(), FormatError> { + let n = match codec { + Codec::BloscLz => blosclz_decompress(src, dst), + Codec::Lz4 => { + lz4_flex::block::decompress_into(src, dst).map_err(|e| err(&format!("lz4: {e}")))? + } + Codec::Snappy => { + let len = snap::raw::decompress_len(src).map_err(|e| err(&format!("snappy: {e}")))?; + if len != dst.len() { + return Err(err("snappy stream has the wrong size")); + } + snap::raw::Decoder::new() + .decompress(src, dst) + .map_err(|e| err(&format!("snappy: {e}")))? + } + Codec::Zlib => { + let out = crate::filters::inflate_bounded(src, dst.len(), dst.len()) + .map_err(|e| err(&format!("zlib: {e}")))?; + let n = out.len(); + if n == dst.len() { + dst.copy_from_slice(&out); + } + n + } + Codec::Zstd => crate::filters_bitshuffle::zstd_decode_into( + zstd.get_or_insert_with(ruzstd::decoding::FrameDecoder::new), + src, + dst, + )?, + }; + if n != dst.len() { + return Err(err("stream decoded to the wrong size")); + } + Ok(()) +} + +/// Decode a Blosc-filtered chunk: one Blosc 1 frame. +pub(crate) fn blosc_decode(input: &[u8], ctx: &FilterContext<'_>) -> Result, FormatError> { + blosc_decompress(input, ctx.output_limit()) +} + +/// Decompress a Blosc 1 frame, refusing more than `limit` bytes of output. +pub fn blosc_decompress(input: &[u8], limit: usize) -> Result, FormatError> { + if input.len() < HEADER { + return Err(err("truncated header")); + } + let version = input[0]; + let codec_version = input[1]; + let flags = input[2]; + let typesize = input[3] as usize; + let nbytes = le32(input, 4)?; + let blocksize = le32(input, 8)?; + let cbytes = le32(input, 12)?; + if version != 1 && version != 2 { + return Err(err(&format!( + "frame format version {version} is not Blosc 1 (a Blosc 2 chunk?)" + ))); + } + if flags & FLAG_FUTURE != 0 { + return Err(err("unknown header flags")); + } + if nbytes > limit { + return Err(err("decoded size exceeds the chunk size")); + } + if cbytes > input.len() { + return Err(err("frame is longer than the chunk")); + } + let src = &input[..cbytes]; + if nbytes == 0 { + return Ok(Vec::new()); + } + if blocksize == 0 || typesize == 0 { + return Err(err("bad block or type size")); + } + let mut out = vec![0u8; nbytes]; + if flags & FLAG_MEMCPYED != 0 { + if cbytes != nbytes + HEADER { + return Err(err("stored frame has the wrong size")); + } + out.copy_from_slice(&src[HEADER..]); + return Ok(out); + } + let codec = Codec::from_flags(flags)?; + if codec_version != 1 { + return Err(err(&format!( + "unsupported {codec:?} format version {codec_version}" + ))); + } + let nblocks = nbytes.div_ceil(blocksize); + let leftover = nbytes % blocksize; + if nblocks > (cbytes - HEADER) / 4 { + return Err(err("block table is truncated")); + } + let block_len = blocksize.min(nbytes); + let mut tmp = vec![0u8; block_len]; + let mut zstd = None; + let dont_split = flags & FLAG_DONT_SPLIT != 0; + for j in 0..nblocks { + let is_leftover = j == nblocks - 1 && leftover > 0; + let bsize = if is_leftover { leftover } else { blocksize }; + let nsplits = if !dont_split + && typesize <= MAX_SPLITS + && bsize / typesize >= MIN_BUFFERSIZE + && !is_leftover + { + typesize + } else { + 1 + }; + let neblock = bsize / nsplits; + let mut pos = le32(src, HEADER + 4 * j)?; + let tmp = &mut tmp[..bsize]; + for s in 0..nsplits { + if pos + 4 > src.len() { + return Err(err("block offset out of range")); + } + let clen = le32(src, pos)?; + pos += 4; + let stream = src + .get(pos..pos.saturating_add(clen)) + .ok_or_else(|| err("stream runs past the frame"))?; + let dst = &mut tmp[s * neblock..(s + 1) * neblock]; + if clen == neblock { + dst.copy_from_slice(stream); + } else { + decode_stream(codec, stream, dst, &mut zstd)?; + } + pos += clen; + } + // `bsize` is a whole number of splits by construction (`nsplits` > 1 + // only for full blocks, and c-blosc sizes those in whole elements). + if nsplits * neblock != bsize { + return Err(err("block is not a whole number of streams")); + } + let dest = &mut out[j * blocksize..j * blocksize + bsize]; + unshuffle_block(flags, typesize, tmp, dest); + } + Ok(out) +} + +/// Undo the frame's shuffle on one decoded block. +fn unshuffle_block(flags: u8, typesize: usize, src: &[u8], dest: &mut [u8]) { + let bsize = src.len(); + if flags & FLAG_SHUFFLE != 0 && typesize > 1 { + let n = bsize / typesize; + for i in 0..n { + for b in 0..typesize { + dest[i * typesize + b] = src[b * n + i]; + } + } + dest[n * typesize..].copy_from_slice(&src[n * typesize..]); + } else if flags & FLAG_BITSHUFFLE != 0 && bsize >= typesize { + let n = bsize / typesize; + if n.is_multiple_of(8) { + let body = n * typesize; + bitunshuffle_block(&src[..body], &mut dest[..body], n, typesize); + dest[body..].copy_from_slice(&src[body..]); + } else { + dest.copy_from_slice(src); + } + } else { + dest.copy_from_slice(src); + } +} + +/// BloscLZ decompression (c-blosc 1.21 `blosclz_decompress`): returns the +/// number of bytes written, or 0 on malformed input — exactly as the C +/// decoder, including stopping before a match that ends the stream, so a +/// stream libblosc rejects is rejected here too. +/// +/// Instructions: a control byte `ctrl`. Below 32, a literal run of +/// `ctrl + 1` bytes. Otherwise a match: length `(ctrl >> 5) + 2`, extended +/// by following bytes while they are 255 when the top three bits are all +/// set; distance `((ctrl & 31) << 8) + next byte + 1`, or — when that byte +/// is 255 and the high bits are 31 — a 16-bit big-endian distance plus 8192. +/// The first instruction is always a literal. +pub(crate) fn blosclz_decompress(input: &[u8], out: &mut [u8]) -> usize { + const MAX_DISTANCE: usize = 8191; + let limit = input.len(); + if limit == 0 { + return 0; + } + let mut ip = 1usize; + let mut op = 0usize; + let mut ctrl = (input[0] & 31) as usize; + loop { + if ctrl >= 32 { + let mut len = (ctrl >> 5) - 1; + let ofs = (ctrl & 31) << 8; + if len == 6 { + loop { + if ip + 1 >= limit { + return 0; + } + let code = input[ip] as usize; + ip += 1; + len += code; + if code != 255 { + break; + } + } + } else if ip + 1 >= limit { + return 0; + } + let code = input[ip] as usize; + ip += 1; + len += 3; + // The copy source is `distance` bytes back. + let mut distance = ofs + code + 1; + if code == 255 && ofs == 31 << 8 { + if ip + 1 >= limit { + return 0; + } + let far = ((input[ip] as usize) << 8) + input[ip + 1] as usize; + ip += 2; + distance = far + MAX_DISTANCE + 1; + } + if op + len > out.len() { + return 0; + } + if distance > op { + return 0; + } + if ip >= limit { + break; + } + ctrl = input[ip] as usize; + ip += 1; + let start = op - distance; + if distance >= len { + out.copy_within(start..start + len, op); + } else { + for k in 0..len { + out[op + k] = out[start + k]; + } + } + op += len; + } else { + let run = ctrl + 1; + if op + run > out.len() || ip + run > limit { + return 0; + } + out[op..op + run].copy_from_slice(&input[ip..ip + run]); + op += run; + ip += run; + if ip >= limit { + break; + } + ctrl = input[ip] as usize; + ip += 1; + } + } + op +} + +/// The codec our encoder puts inside the frame. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum EncodeCodec { + Lz4, + Snappy, + Zlib, + Zstd, +} + +impl EncodeCodec { + /// From the filter's `cd_values[6]` compressor code. + fn from_cd(code: u32) -> Result { + match code { + 1 | 2 => Ok(EncodeCodec::Lz4), + 3 => Ok(EncodeCodec::Snappy), + 4 => Ok(EncodeCodec::Zlib), + 5 => Ok(EncodeCodec::Zstd), + 0 => Err(FormatError::CompressionError( + "blosc: clawhdf5 cannot write BloscLZ; choose lz4, snappy, zlib or zstd".into(), + )), + other => Err(FormatError::CompressionError(format!( + "blosc: unknown compressor {other}" + ))), + } + } + + fn flags(self) -> u8 { + (match self { + EncodeCodec::Lz4 => 1, + EncodeCodec::Snappy => 2, + EncodeCodec::Zlib => 3, + EncodeCodec::Zstd => 4, + }) << 5 + } + + fn encode(self, data: &[u8], level: u32) -> Result, FormatError> { + match self { + EncodeCodec::Lz4 => Ok(lz4_flex::block::compress(data)), + EncodeCodec::Snappy => snap::raw::Encoder::new() + .compress_vec(data) + .map_err(|e| FormatError::CompressionError(format!("blosc: snappy: {e}"))), + EncodeCodec::Zlib => crate::filters::deflate_bounded(data, level.min(9)) + .map_err(|e| FormatError::CompressionError(format!("blosc: zlib: {e}"))), + EncodeCodec::Zstd => Ok(crate::filters_bitshuffle::zstd_encode(data)), + } + } +} + +/// Block size our encoder uses: at most 256 KiB, a whole number of +/// elements (and, for bit shuffle, of 8-element groups). +fn encode_block_size(nbytes: usize, typesize: usize, bitshuffle: bool) -> usize { + let unit = if bitshuffle { 8 * typesize } else { typesize }; + let target = (256 * 1024).min(nbytes); + if target < unit { + return nbytes.max(1); + } + target / unit * unit +} + +/// Encode a chunk as one Blosc 1 frame. `cd_values` as hdf5-blosc: +/// `[2]` type size, `[4]` level (0 = store), `[5]` shuffle, `[6]` codec. +pub(crate) fn blosc_encode(input: &[u8], ctx: &FilterContext<'_>) -> Result, FormatError> { + let cd = ctx.client_data(); + let cerr = |m: &str| FormatError::CompressionError(format!("blosc: {m}")); + let typesize = match cd.get(2) { + Some(&t) if t != 0 => t as usize, + _ => ctx.element_size.max(1), + }; + // Blosc records the type size in one byte; c-blosc treats larger types + // as bytes. + let typesize = if typesize > 255 { 1 } else { typesize }; + let level = cd.get(4).copied().unwrap_or(5); + let shuffle = cd.get(5).copied().unwrap_or(1); + let codec = EncodeCodec::from_cd(cd.get(6).copied().unwrap_or(1))?; + let nbytes = input.len(); + if nbytes > i32::MAX as usize - HEADER { + return Err(cerr("chunk too large for a Blosc frame")); + } + let mut flags = codec.flags(); + match shuffle { + 0 => {} + 1 => flags |= FLAG_SHUFFLE, + 2 => flags |= FLAG_BITSHUFFLE, + other => return Err(cerr(&format!("unknown shuffle mode {other}"))), + } + let blocksize = encode_block_size(nbytes, typesize, shuffle == 2); + let header = |flags: u8, blocksize: usize, cbytes: usize| { + let mut h = Vec::with_capacity(HEADER); + h.extend_from_slice(&[2, 1, flags, typesize as u8]); + h.extend_from_slice(&(nbytes as u32).to_le_bytes()); + h.extend_from_slice(&(blocksize as u32).to_le_bytes()); + h.extend_from_slice(&(cbytes as u32).to_le_bytes()); + h + }; + let stored = || { + let mut out = header( + FLAG_MEMCPYED | (flags & !(FLAG_SHUFFLE | FLAG_BITSHUFFLE)), + blocksize, + nbytes + HEADER, + ); + out.extend_from_slice(input); + out + }; + if level == 0 || nbytes == 0 { + return Ok(stored()); + } + + let nblocks = nbytes.div_ceil(blocksize); + let leftover = nbytes % blocksize; + let mut body = Vec::with_capacity(nbytes / 2); + let mut starts = Vec::with_capacity(nblocks); + let table_end = HEADER + 4 * nblocks; + let mut shuffled = vec![0u8; blocksize]; + for j in 0..nblocks { + let is_leftover = j == nblocks - 1 && leftover > 0; + let bsize = if is_leftover { leftover } else { blocksize }; + let block = &input[j * blocksize..j * blocksize + bsize]; + let sh = &mut shuffled[..bsize]; + shuffle_block(flags, typesize, block, sh); + starts.push(table_end + body.len()); + let nsplits = if typesize <= MAX_SPLITS + && bsize / typesize >= MIN_BUFFERSIZE + && !is_leftover + && bsize.is_multiple_of(typesize) + { + typesize + } else { + 1 + }; + let neblock = bsize / nsplits; + for s in 0..nsplits { + let part = &sh[s * neblock..(s + 1) * neblock]; + let comp = codec.encode(part, level)?; + if comp.len() < neblock { + body.extend_from_slice(&(comp.len() as u32).to_le_bytes()); + body.extend_from_slice(&comp); + } else { + body.extend_from_slice(&(neblock as u32).to_le_bytes()); + body.extend_from_slice(part); + } + } + if table_end + body.len() >= nbytes + HEADER { + // Incompressible: store instead, as c-blosc does. + return Ok(stored()); + } + } + // A split block must decode as split: the decoder infers splitting from + // the same rule, which requires a whole number of elements per block. + let cbytes = table_end + body.len(); + let mut out = header(flags, blocksize, cbytes); + for s in starts { + out.extend_from_slice(&(s as u32).to_le_bytes()); + } + out.extend_from_slice(&body); + Ok(out) +} + +/// Apply the frame's shuffle to one block (the inverse of +/// [`unshuffle_block`]). +fn shuffle_block(flags: u8, typesize: usize, src: &[u8], dest: &mut [u8]) { + let bsize = src.len(); + if flags & FLAG_SHUFFLE != 0 && typesize > 1 { + let n = bsize / typesize; + for i in 0..n { + for b in 0..typesize { + dest[b * n + i] = src[i * typesize + b]; + } + } + dest[n * typesize..].copy_from_slice(&src[n * typesize..]); + } else if flags & FLAG_BITSHUFFLE != 0 && bsize >= typesize { + let n = bsize / typesize; + if n.is_multiple_of(8) { + let body = n * typesize; + bitshuffle_block(&src[..body], &mut dest[..body], n, typesize); + dest[body..].copy_from_slice(&src[body..]); + } else { + dest.copy_from_slice(src); + } + } else { + dest.copy_from_slice(src); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::filter_pipeline::{FILTER_BLOSC, FilterDescription}; + + /// A blosclz stream: literal "abc", then a 9-byte match 3 back (a run + /// of "abc"), then literal "Z". + #[test] + fn blosclz_decodes_literals_and_overlapping_matches() { + // Match: length (ctrl >> 5) + 2 = 8, distance ofs + code + 1 = 3. + let stream = [2, b'a', b'b', b'c', (6 << 5), 2, 0, b'Z']; + let mut out = [0u8; 12]; + assert_eq!(blosclz_decompress(&stream, &mut out), 12); + assert_eq!(&out, b"abcabcabcabZ"); + // A stream cut inside a match is malformed. + let mut out = [0u8; 11]; + assert_eq!(blosclz_decompress(&stream[..6], &mut out), 0); + // A match before the start of the output is malformed. + assert_eq!(blosclz_decompress(&[0, b'a', 32, 5, 0, b'x'], &mut out), 0); + } + + fn desc(cd: Vec) -> FilterDescription { + FilterDescription { + filter_id: FILTER_BLOSC, + name: None, + flags: 0, + client_data: cd, + } + } + + #[test] + fn frame_round_trips_every_codec_and_shuffle() { + for ts in [1usize, 2, 4, 8, 3, 32] { + for n in [0usize, 5, 100, 1000, 70_000, 300_001] { + if n * ts > 1 << 20 && ts > 1 { + continue; + } + let data: Vec = (0..n * ts) + .map(|i| ((i / ts) % 200) as u8 ^ (i % ts) as u8) + .collect(); + for codec in [1u32, 3, 4, 5] { + for shuffle in [0u32, 1, 2] { + for level in [0u32, 5] { + let f = desc(vec![2, 2, ts as u32, 0, level, shuffle, codec]); + let ctx = FilterContext { + filter: &f, + element_size: ts, + max_output: data.len(), + }; + let enc = blosc_encode(&data, &ctx).unwrap(); + let dec = blosc_decode(&enc, &ctx).unwrap_or_else(|e| { + panic!("ts={ts} n={n} codec={codec} shuffle={shuffle}: {e}") + }); + assert!( + dec == data, + "ts={ts} n={n} codec={codec} shuffle={shuffle} level={level}" + ); + } + } + } + } + } + } + + #[test] + fn rejects_bad_frames() { + let data = vec![9u8; 50_000]; + let f = desc(vec![2, 2, 4, 0, 5, 1, 1]); + let ctx = FilterContext { + filter: &f, + element_size: 4, + max_output: data.len(), + }; + let enc = blosc_encode(&data, &ctx).unwrap(); + assert!(blosc_decode(&enc[..enc.len() - 3], &ctx).is_err()); + let small = FilterContext { + max_output: 49_999, + ..ctx + }; + assert!(blosc_decode(&enc, &small).is_err()); + let mut v3 = enc.clone(); + v3[0] = 3; + assert!(blosc_decode(&v3, &ctx).is_err()); + let f0 = desc(vec![2, 2, 4, 0, 5, 1, 0]); + let ctx0 = FilterContext { filter: &f0, ..ctx }; + assert!(blosc_encode(&data, &ctx0).is_err()); + } +} diff --git a/crates/clawhdf5-format/src/lib.rs b/crates/clawhdf5-format/src/lib.rs index b27ca09..d9bba65 100644 --- a/crates/clawhdf5-format/src/lib.rs +++ b/crates/clawhdf5-format/src/lib.rs @@ -73,8 +73,10 @@ pub mod fill_value; pub mod filter_pipeline; pub mod filter_registry; pub mod filters; -#[cfg(feature = "bitshuffle")] +#[cfg(any(feature = "bitshuffle", feature = "blosc"))] mod filters_bitshuffle; +#[cfg(feature = "blosc")] +pub mod filters_blosc; #[cfg(feature = "bzip2")] mod filters_bzip2; #[cfg(feature = "lzf")] diff --git a/crates/clawhdf5-format/src/type_builders.rs b/crates/clawhdf5-format/src/type_builders.rs index 635d66a..f055a88 100644 --- a/crates/clawhdf5-format/src/type_builders.rs +++ b/crates/clawhdf5-format/src/type_builders.rs @@ -770,6 +770,22 @@ impl DatasetBuilder { self.with_plugin_filter(crate::chunked_write::PluginFilter::Bzip2 { level }) } + /// Enable Blosc (filter 32001) with `codec` at `level` (0-9) after + /// `shuffle`. Implies chunked storage; no extra HDF5 shuffle is added. + /// Requires the `blosc` cargo feature. + pub fn with_blosc( + &mut self, + codec: crate::chunked_write::BloscCodec, + level: u32, + shuffle: crate::chunked_write::BloscShuffle, + ) -> &mut Self { + self.with_plugin_filter(crate::chunked_write::PluginFilter::Blosc { + codec, + level, + shuffle, + }) + } + /// Enable Pcodec lossless numerical compression (private clawhdf5 filter /// ID 480). /// diff --git a/crates/clawhdf5/Cargo.toml b/crates/clawhdf5/Cargo.toml index 1523920..a6401fe 100644 --- a/crates/clawhdf5/Cargo.toml +++ b/crates/clawhdf5/Cargo.toml @@ -46,6 +46,9 @@ pcodec = ["clawhdf5-format/pcodec"] lzf = ["clawhdf5-format/lzf"] bitshuffle = ["clawhdf5-format/bitshuffle"] bzip2 = ["clawhdf5-format/bzip2"] +blosc = ["clawhdf5-format/blosc"] +# Every plugin filter. +plugin-filters = ["lzf", "bitshuffle", "bzip2", "blosc"] # Dataset::verify_provenance() — recompute a dataset's SHA-256 and compare # against its stored _provenance_sha256 attribute. On by default, matching # clawhdf5-format's own default-on `provenance` feature. diff --git a/crates/clawhdf5/tests/plugin_filters_interop.rs b/crates/clawhdf5/tests/plugin_filters_interop.rs index 5cd1188..fc44e92 100644 --- a/crates/clawhdf5/tests/plugin_filters_interop.rs +++ b/crates/clawhdf5/tests/plugin_filters_interop.rs @@ -76,7 +76,7 @@ try: except ImportError: hdf5plugin = None path = sys.argv[1] -FILTERS = eval(sys.argv[2]) +FILTERS = eval('(' + sys.argv[2] + ')') cases = [ (' Date: Sat, 26 Sep 2026 00:11:57 -0500 Subject: [PATCH 06/13] feat(format): name the filter in UnsupportedFilter errors; Blosc2/ZFP stay errors Blosc2 (32026) is out of reach for now: hdf5plugin's Blosc2 filter stores each HDF5 chunk as a Blosc2 super-chunk frame (msgpack header, a compressed chunk-offset index, trailer metalayers) and, for 2-D and larger chunks, as a B2ND array whose n-D blocks have to be reassembled - on top of the Blosc2 chunk format itself (extended header, filter pipeline, special-value chunks). ZFP (32013) is out of scope. Both keep failing with UnsupportedFilter, and the message now says what the ID is: "unsupported filter: 32026 (Blosc2, not implemented by clawhdf5)", or, for a filter this build left out, "... (Blosc; this build lacks the `blosc` feature)". filter_registry::known_filter exposes the table. tests/plugin_filters_interop.rs: hdf5plugin's Blosc2 and ZFP datasets read as an error naming the filter, never as data. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/error.rs | 14 +++++-- crates/clawhdf5-format/src/filter_registry.rs | 41 +++++++++++++++++++ .../clawhdf5/tests/plugin_filters_interop.rs | 35 ++++++++++++++++ 3 files changed, 87 insertions(+), 3 deletions(-) diff --git a/crates/clawhdf5-format/src/error.rs b/crates/clawhdf5-format/src/error.rs index bb81939..b4c3f18 100644 --- a/crates/clawhdf5-format/src/error.rs +++ b/crates/clawhdf5-format/src/error.rs @@ -406,9 +406,17 @@ impl fmt::Display for FormatError { FormatError::InvalidFilterPipelineVersion(v) => { write!(f, "invalid filter pipeline version: {v}") } - FormatError::UnsupportedFilter(id) => { - write!(f, "unsupported filter: {id}") - } + FormatError::UnsupportedFilter(id) => match crate::filter_registry::known_filter(*id) { + Some((name, Some(feature))) => write!( + f, + "unsupported filter: {id} ({name}; this build lacks the `{feature}` feature)" + ), + Some((name, None)) => write!( + f, + "unsupported filter: {id} ({name}, not implemented by clawhdf5)" + ), + None => write!(f, "unsupported filter: {id}"), + }, FormatError::FilterError(msg) => { write!(f, "filter error: {msg}") } diff --git a/crates/clawhdf5-format/src/filter_registry.rs b/crates/clawhdf5-format/src/filter_registry.rs index acbf71a..62e22b8 100644 --- a/crates/clawhdf5-format/src/filter_registry.rs +++ b/crates/clawhdf5-format/src/filter_registry.rs @@ -130,6 +130,30 @@ pub fn builtin_filter(id: u16) -> Option<&'static BuiltinFilter> { builtin_filters().iter().find(|f| f.id == id) } +/// Why a filter ID may be missing from this build: the filter's name, and +/// the cargo feature that provides it (`None`: clawhdf5 does not implement +/// it — register a codec for it with [`register_filter`]). `None` for an ID +/// clawhdf5 knows nothing about. +pub fn known_filter(id: u16) -> Option<(&'static str, Option<&'static str>)> { + Some(match id { + 1 => ("deflate", Some("deflate")), + 4 => ("SZIP", Some("szip")), + 307 => ("bzip2", Some("bzip2")), + 480 => ("pcodec", Some("pcodec")), + 32000 => ("LZF", Some("lzf")), + 32001 => ("Blosc", Some("blosc")), + 32004 => ("LZ4", Some("lz4")), + 32008 => ("bitshuffle", Some("bitshuffle")), + 32013 => ("ZFP", None), + 32015 => ("Zstandard", Some("zstd")), + 32019 => ("JPEG", None), + 32022 => ("BitGroom", None), + 32023 => ("Granular BitRound", None), + 32026 => ("Blosc2", None), + _ => return None, + }) +} + /// Whether a chunk filtered with `id` can be decoded: a built-in filter or a /// registered one. pub fn is_filter_available(id: u16) -> bool { @@ -344,6 +368,23 @@ mod tests { assert!(builtin_filter(FILTER_SHUFFLE).is_some()); } + #[test] + fn unsupported_filter_error_names_the_filter() { + let msg = FormatError::UnsupportedFilter(32026).to_string(); + assert!( + msg.contains("Blosc2") && msg.contains("not implemented"), + "{msg}" + ); + let msg = FormatError::UnsupportedFilter(32013).to_string(); + assert!(msg.contains("ZFP"), "{msg}"); + let msg = FormatError::UnsupportedFilter(32000).to_string(); + assert!(msg.contains("LZF") && msg.contains("`lzf`"), "{msg}"); + assert_eq!( + FormatError::UnsupportedFilter(399).to_string(), + "unsupported filter: 399" + ); + } + #[test] fn builtin_table_is_sorted_and_unique() { let ids: Vec = builtin_filters().iter().map(|f| f.id).collect(); diff --git a/crates/clawhdf5/tests/plugin_filters_interop.rs b/crates/clawhdf5/tests/plugin_filters_interop.rs index fc44e92..647c007 100644 --- a/crates/clawhdf5/tests/plugin_filters_interop.rs +++ b/crates/clawhdf5/tests/plugin_filters_interop.rs @@ -380,3 +380,38 @@ fn lzf_written_by_clawhdf5_reads_in_h5py() { ds.with_lzf().without_shuffle(); }); } + +/// Blosc2 and ZFP are not implemented: reading them must be a clear error +/// naming the filter, never data. +#[test] +fn unimplemented_filters_are_a_clear_error() { + if !have_python("h5py, hdf5plugin") { + return; + } + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("unimplemented.h5"); + run_python( + r#" +import sys +import numpy as np, h5py, hdf5plugin +with h5py.File(sys.argv[1], 'w') as f: + d = np.arange(4096, dtype=' Date: Sat, 26 Sep 2026 00:13:50 -0500 Subject: [PATCH 07/13] ci: lint and test the plugin filters; the conformance probe reads them scripts/ci-test.sh: the format feature matrix (clippy and tests) adds plugin-filters; bitshuffle, bzip2 and blosc are each linted alone (blosc and bitshuffle share code); the facade is linted with plugin-filters; and the interop section runs tests/plugin_filters_interop.rs with it, against h5py + hdf5plugin (CI's venv already installs hdf5plugin). conformance/probe enables plugin-filters. Sweep (tank, 2026-09-26, conformance/run.sh --no-fetch against the cached corpus): 573 of 697 ok (baseline 569), no regressions; newly ok: h5ex_d_blosc.h5, h5ex_d_bshuf.h5, h5ex_d_bzip2.h5, h5ex_d_lzf.h5. h5ex_d_blosc2.h5 and h5ex_d_zfp.h5 remain UnsupportedFilter. Co-Authored-By: Claude Opus 5.5 (1M context) --- conformance/probe/Cargo.lock | 33 +++++++++++++++++++++++++++++++++ conformance/probe/Cargo.toml | 2 +- scripts/ci-test.sh | 26 ++++++++++++++++++++++++-- 3 files changed, 58 insertions(+), 3 deletions(-) diff --git a/conformance/probe/Cargo.lock b/conformance/probe/Cargo.lock index 0ffb1f7..964e0ab 100644 --- a/conformance/probe/Cargo.lock +++ b/conformance/probe/Cargo.lock @@ -29,6 +29,15 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" +[[package]] +name = "bzip2" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3a53fac24f34a81bc9954b5d6cfce0c21e18ec6959f44f56e8e90e4bb7c346c" +dependencies = [ + "libbz2-rs-sys", +] + [[package]] name = "cc" version = "1.5.1" @@ -52,12 +61,15 @@ name = "clawhdf5-format" version = "2.7.0" dependencies = [ "byteorder", + "bzip2", "flate2", "libaec-sys", "lz4_flex", "pco", "portable-atomic", + "ruzstd", "sha2", + "snap", "zstd", ] @@ -192,6 +204,12 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "libbz2-rs-sys" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34b357333733e8260735ba5894eb928c02ecc69c78715f01a8019e7fa7f2db4c" + [[package]] name = "libc" version = "0.2.189" @@ -286,6 +304,15 @@ dependencies = [ "rand_core", ] +[[package]] +name = "ruzstd" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a252f5e20f038fe7b4ea53e073e65398d652c864cc162fc77c56c2f13717b888" +dependencies = [ + "twox-hash", +] + [[package]] name = "serde" version = "1.0.229" @@ -351,6 +378,12 @@ version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" +[[package]] +name = "snap" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "199905e6153d6405f9728fe44daace35f8f837bbf830bb6e85fbd5828709a886" + [[package]] name = "syn" version = "2.0.119" diff --git a/conformance/probe/Cargo.toml b/conformance/probe/Cargo.toml index f5c68af..62470b2 100644 --- a/conformance/probe/Cargo.toml +++ b/conformance/probe/Cargo.toml @@ -12,7 +12,7 @@ description = "Walks an HDF5 file with clawhdf5-format and prints a canonical JS [workspace] [dependencies] -clawhdf5-format = { path = "../../crates/clawhdf5-format", features = ["lz4", "zstd", "szip", "pcodec"] } +clawhdf5-format = { path = "../../crates/clawhdf5-format", features = ["lz4", "zstd", "szip", "pcodec", "plugin-filters"] } serde_json = "1" sha2 = "0.10" diff --git a/scripts/ci-test.sh b/scripts/ci-test.sh index 1d7a7f9..dedb2b4 100755 --- a/scripts/ci-test.sh +++ b/scripts/ci-test.sh @@ -64,10 +64,29 @@ run_step "cargo clippy --all-targets" cargo clippy \ # 3. Clippy over clawhdf5-format's optional features, which the default # workspace build never compiles (szip is left out: it needs libaec). +# plugin-filters = bitshuffle, bzip2, blosc (and the default-on lzf). run_step "cargo clippy (format feature matrix)" cargo clippy \ -p clawhdf5-format \ --all-targets \ - --features parallel,lz4,zstd,pcodec,fast-checksum \ + --features parallel,lz4,zstd,pcodec,fast-checksum,plugin-filters \ + -- -D warnings + +# Each plugin filter alone, so none of them leans on another's +# dependencies (bitshuffle and blosc share code). +plugin_filters_alone() { + local f + for f in bitshuffle bzip2 blosc; do + echo "--- $f" + cargo clippy -p clawhdf5-format --all-targets --features "$f" -- -D warnings || return 1 + done +} +run_step "cargo clippy (each plugin filter alone)" plugin_filters_alone + +# The facade's plugin-filter interop tests only build with the features on. +run_step "cargo clippy (facade plugin filters)" cargo clippy \ + -p clawhdf5 \ + --all-targets \ + --features plugin-filters \ -- -D warnings # The HNSW index's parallel bulk build is feature-gated too. @@ -128,7 +147,7 @@ run_step "cargo test" cargo test \ run_step "cargo test (format feature matrix)" cargo test \ -p clawhdf5-format \ - --features parallel,lz4,zstd,pcodec,fast-checksum + --features parallel,lz4,zstd,pcodec,fast-checksum,plugin-filters run_step "cargo test (ann parallel)" cargo test \ -p clawhdf5-ann \ @@ -149,6 +168,9 @@ if "$PYTHON" -c "import h5py" >/dev/null 2>&1 || [ "${CLAWHDF5_REQUIRE_INTEROP:- # libhdf5's registered plugins) compile and run too. run_step "h5py interop (format, ignored tests)" cargo test \ -p clawhdf5-format --features lz4,zstd --test writer_h5py_tests -- --include-ignored + # LZF, bitshuffle, bzip2 and Blosc both ways against h5py + hdf5plugin. + run_step "h5py interop (plugin filters)" cargo test \ + -p clawhdf5 --features plugin-filters --test plugin_filters_interop else echo "" echo "==> [h5py interop] SKIPPED: no h5py in $PYTHON" From e01160299add4a488a4dcd97d842af234bf9290e Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 00:14:47 -0500 Subject: [PATCH 08/13] docs: plugin filters and the filter registry CHANGELOG (Unreleased): LZF, bitshuffle, bzip2 and Blosc read and write in pure Rust, their features, the ChunkOptions::plugin field (breaking for struct-literal construction), the filter registry, the named UnsupportedFilter message, and Blosc2/ZFP still unimplemented. README: the clawhdf5-format feature table gains lzf (default), bitshuffle, bzip2, blosc and plugin-filters, with how to write them and what is not implemented; no speed claims. docs/known-issues.md: the audit's filter gap is marked fixed 2026-09-26 for LZF/bitshuffle/bzip2/Blosc, Blosc2 and ZFP still open. CLAUDE.md: the clawhdf5-filters row no longer says "No Blosc". clawhdf5-format's crate docs list the new features. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 30 ++++++++++++++++++++++++++++++ CLAUDE.md | 2 +- README.md | 18 +++++++++++++++++- crates/clawhdf5-format/src/lib.rs | 8 ++++++++ docs/known-issues.md | 9 ++++++++- 5 files changed, 64 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1fc429d..9318dbc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,36 @@ ## Unreleased +### Plugin filters (2026-09-26) +- **LZF, bitshuffle, bzip2 and Blosc read and write, in pure Rust.** Files + written by h5py with `compression="lzf"`, or with hdf5plugin's + `Bitshuffle`, `BZip2` and `Blosc`, failed with `UnsupportedFilter`. New + `clawhdf5-format`/`clawhdf5` features: `lzf` (32000, **on by default**, no + dependencies), `bitshuffle` (32008: transpose only, LZ4 and Zstandard + modes), `bzip2` (307), `blosc` (32001: Blosc 1 frames with BloscLZ, + LZ4/LZ4HC, Snappy, Zlib and Zstandard codecs and byte/bit shuffle; + BloscLZ is decoded by a port of c-blosc 1.21's decoder, and cannot be + written), and `plugin-filters` for all four. None compiles C: Zstandard is + ruzstd, bzip2 is libbz2-rs-sys. Write with `DatasetBuilder::with_lzf()`, + `with_bitshuffle(..)`, `with_bzip2(..)`, `with_blosc(..)` or + `with_plugin_filter(PluginFilter::..)`; `ChunkOptions` gains a `plugin` + field (**breaking** for code that builds `ChunkOptions` with a struct + literal and no `..Default::default()`). Tested both ways against h5py 3.16 + + hdf5plugin 7.1 over 1-3-D shapes with partial edge chunks, 1-8-byte + types in both byte orders and incompressible data + (`crates/clawhdf5/tests/plugin_filters_interop.rs`). Conformance: 573 of + 697 files ok (was 569) — h5ex_d_lzf/bshuf/bzip2/blosc. +- **Filter registry.** Filters are looked up by ID in + `clawhdf5_format::filter_registry` instead of a `match`: the built-in + table (per build), then codecs registered at run time with + `register_filter(id, codec)` — a decoding closure or a `FilterCodec` that + can also encode. Built-in IDs cannot be overridden; a registered decoder's + output is held to the chunk-size bound. Unknown IDs still fail with + `UnsupportedFilter(id)`, whose message now names known filters and the + missing feature ("unsupported filter: 32026 (Blosc2, not implemented by + clawhdf5)"). +- **Not implemented:** Blosc2 (32026) and ZFP (32013) remain a clear error. + ### Upgrade Notes - **HDF5 correctness audit (2026-09-25).** A sweep of 686 public files (the libhdf5 test files, the HDF Group's CVE reproducers, pyfive, netcdf-c, diff --git a/CLAUDE.md b/CLAUDE.md index 3f8d9c1..12a4b93 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -11,7 +11,7 @@ Cargo workspace with 16 crates under `crates/` (plus `libaec-sys`, an internal F |-------|------| | `clawhdf5-format` | HDF5 binary spec parser (superblock, B-tree, heap) — also holds shared type definitions and physical constants | | `clawhdf5-io` | Read/write implementation | -| `clawhdf5-filters` | Deflate backends (zlib-rs, zlib-ng, Apple Compression); the HDF5 filter pipeline and the other codecs (LZ4, Zstd, SZIP, N-Bit, scale-offset, pcodec) live in `clawhdf5-format`. No Blosc. | +| `clawhdf5-filters` | Deflate backends (zlib-rs, zlib-ng, Apple Compression); the HDF5 filter pipeline, the filter registry (`clawhdf5_format::filter_registry`) and the other codecs (LZ4, Zstd, SZIP, N-Bit, scale-offset, pcodec, and the pure-Rust plugin filters LZF, bitshuffle, bzip2, Blosc 1) live in `clawhdf5-format`. No Blosc2 or ZFP. | | `clawhdf5-derive` | Proc-macro derive for HDF5-serializable structs | | `clawhdf5` | Main facade crate | | `clawhdf5-netcdf4` | NetCDF-4 compatibility layer | diff --git a/README.md b/README.md index 5d4c707..1088194 100644 --- a/README.md +++ b/README.md @@ -594,7 +594,7 @@ clawhdf5 workspace (16 crates, ~86K lines of Rust in src/, ~104K with tests ├── Core HDF5 │ ├── clawhdf5-format — Binary parser/writer (no_std-capable), shared type definitions │ ├── clawhdf5-io — I/O abstraction (file/memory readers; optional mmap, async, HSDS, MPI) -│ ├── clawhdf5-filters — Fast deflate path (zlib-ng); lz4/zstd/pcodec/szip filters live in clawhdf5-format +│ ├── clawhdf5-filters — Fast deflate path (zlib-ng); the filter registry and the lz4/zstd/pcodec/szip/LZF/bitshuffle/bzip2/Blosc filters live in clawhdf5-format │ ├── clawhdf5-derive — Proc macros │ ├── clawhdf5 — High-level API │ ├── clawhdf5-netcdf4 — NetCDF-4 support @@ -700,6 +700,22 @@ stores keep their setting. Opt out with `float16 = false` or | `system-zlib` | no | System zlib backend for deflate (C) | | `blake3_hash` | no | BLAKE3 content hashing for provenance | | `szip` | no | SZIP filter (id 4) via libaec (C, through the internal `libaec-sys` crate) | +| `lzf` | **yes** | LZF filter (id 32000), h5py's built-in `compression="lzf"`: read and write. No dependencies | +| `bitshuffle` | no | Bitshuffle filter (id 32008) with its LZ4 and Zstandard modes: read and write. Pure Rust (lz4_flex, ruzstd) | +| `bzip2` | no | bzip2 filter (id 307): read and write. Pure Rust (the `bzip2` crate's libbz2-rs-sys backend compiles no C) | +| `blosc` | no | Blosc 1 filter (id 32001): reads BloscLZ, LZ4/LZ4HC, Snappy, Zlib and Zstandard frames with byte or bit shuffle; writes LZ4, Snappy, Zlib or Zstandard (not BloscLZ). Pure Rust | +| `plugin-filters` | no | All four above | + +Blosc2 (32026) and ZFP (32013) are not implemented: reading them fails with +`UnsupportedFilter`, whose message names the filter. Any other filter can be +supplied at run time with `filter_registry::register_filter` (a decoder +closure, or a `FilterCodec` that also encodes). The facade (`clawhdf5`) +forwards `lzf`, `bitshuffle`, `bzip2`, `blosc` and `plugin-filters`. Write +with `DatasetBuilder::with_lzf()`, `with_bitshuffle(..)`, `with_bzip2(..)` +and `with_blosc(..)`; h5py + hdf5plugin read the result (tested both ways in +`crates/clawhdf5/tests/plugin_filters_interop.rs`). The pure-Rust Zstandard +encoder has one level (about zstd's level 1); no speed or ratio claims are +made for these codecs. ### `clawhdf5-ann` diff --git a/crates/clawhdf5-format/src/lib.rs b/crates/clawhdf5-format/src/lib.rs index d9bba65..3e31686 100644 --- a/crates/clawhdf5-format/src/lib.rs +++ b/crates/clawhdf5-format/src/lib.rs @@ -43,6 +43,14 @@ //! | `checksum` | yes | Jenkins lookup3 checksum validation | //! | `deflate` | yes | Deflate (gzip) compression via `flate2` | //! | `provenance` | yes | SHINES provenance — SHA-256 hashing & verification | +//! | `lzf` | yes | LZF filter (32000), h5py's `compression="lzf"` | +//! | `bitshuffle` | no | Bitshuffle filter (32008), none/LZ4/Zstandard | +//! | `bzip2` | no | bzip2 filter (307) | +//! | `blosc` | no | Blosc 1 filter (32001) | +//! | `plugin-filters` | no | The four above | +//! +//! Filters are looked up by ID in [`filter_registry`], which also takes +//! codecs registered at run time for other IDs. #![cfg_attr(not(feature = "std"), no_std)] diff --git a/docs/known-issues.md b/docs/known-issues.md index 4ea7518..53c4996 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -133,7 +133,14 @@ fill-value item that did is fixed). - x87 long double and binary128 are refused. - N-Bit on 64-bit scale-offset data and some N-Bit parameter layouts fail. - **Filters:** blosc, blosc2, bitshuffle, bzip2, LZF and zfp are not - implemented. + implemented. **Fixed 2026-09-26** for LZF (default-on `lzf` feature), + bitshuffle, bzip2 and Blosc 1 (`bitshuffle`, `bzip2`, `blosc`, or + `plugin-filters` for all), read and write, pure Rust; h5ex_d_lzf, + h5ex_d_bshuf, h5ex_d_bzip2 and h5ex_d_blosc now read (conformance 573 of + 697 ok). **Still open:** Blosc2 (32026 — hdf5plugin stores each chunk as a + Blosc2 super-chunk frame, and n-D chunks as B2ND arrays) and ZFP (32013); + both fail with an `UnsupportedFilter` error that names the filter, and + either can be plugged in with `filter_registry::register_filter`. - **Header checks:** on 12 CVE datasets libhdf5 rejects a corrupt header and we read data anyway. We need stricter header checks. - **Writer:** From 9416c5872310046980ce1e3b856fdd2debbbe2d4 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 01:14:33 -0500 Subject: [PATCH 09/13] fix(format): a Blosc frame shorter than its header is an error, not a panic A hostile chunk whose header gave a compressed size below 16 bytes, not stored raw, made the block-table check subtract past zero: a panic in any build with overflow checks (cargo test, maturin develop, debug CLI). The frame size is now checked against the header size, and the stream-length read no longer adds to an untrusted offset. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/filters_blosc.rs | 29 ++++++++++++++++++--- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/crates/clawhdf5-format/src/filters_blosc.rs b/crates/clawhdf5-format/src/filters_blosc.rs index 0730c15..1bf1d15 100644 --- a/crates/clawhdf5-format/src/filters_blosc.rs +++ b/crates/clawhdf5-format/src/filters_blosc.rs @@ -143,6 +143,9 @@ pub fn blosc_decompress(input: &[u8], limit: usize) -> Result, FormatErr if cbytes > input.len() { return Err(err("frame is longer than the chunk")); } + if cbytes < HEADER { + return Err(err("truncated frame")); + } let src = &input[..cbytes]; if nbytes == 0 { return Ok(Vec::new()); @@ -189,10 +192,11 @@ pub fn blosc_decompress(input: &[u8], limit: usize) -> Result, FormatErr let mut pos = le32(src, HEADER + 4 * j)?; let tmp = &mut tmp[..bsize]; for s in 0..nsplits { - if pos + 4 > src.len() { - return Err(err("block offset out of range")); - } - let clen = le32(src, pos)?; + let clen = src + .get(pos..) + .and_then(|rest| rest.get(..4)) + .map(|b| u32::from_le_bytes(b.try_into().unwrap()) as usize) + .ok_or_else(|| err("block offset out of range"))?; pos += 4; let stream = src .get(pos..pos.saturating_add(clen)) @@ -601,4 +605,21 @@ mod tests { let ctx0 = FilterContext { filter: &f0, ..ctx }; assert!(blosc_encode(&data, &ctx0).is_err()); } + + /// A frame whose header claims a compressed size smaller than the + /// header itself, not stored raw: an error, not an arithmetic overflow + /// (it panicked in debug builds). + #[test] + fn frame_size_below_the_header_is_an_error() { + let mut frame = vec![2u8, 1, 1 << 5, 4]; + for v in [64u32, 64, 8] { + frame.extend_from_slice(&v.to_le_bytes()); + } + frame.extend_from_slice(&[0; 40]); + assert!(blosc_decompress(&frame, 1000).is_err()); + for cbytes in 0..16u32 { + frame[12..16].copy_from_slice(&cbytes.to_le_bytes()); + assert!(blosc_decompress(&frame, 1000).is_err(), "cbytes={cbytes}"); + } + } } From 7f52a6f3ba6b43cdaa5283c390aaaf280dca7789 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 01:16:44 -0500 Subject: [PATCH 10/13] test(format): fuzz every plugin-filter decoder for panics Audited LZF, bitshuffle, bzip2 and Blosc/BloscLZ for arithmetic on header fields and unchecked slicing. The only live bug was the Blosc frame-size underflow fixed in the previous commit; bzip2's output-growth step now uses a saturating subtraction as well (the allocator may hand back more capacity than asked for). src/test_fuzz.rs (tests only) feeds each decoder random bytes, truncated seeds and one-to-four-edit mutations of valid frames, biased towards edge-case u32 values in size and offset fields, and asserts no panic and no output over the limit (tests build with overflow checks and debug assertions). Per decoder: LZF, bzip2, bitshuffle in all six mode/block settings plus hostile cd_values, Blosc across four codecs, three shuffles, stored frames and a hand-built BloscLZ frame, and BloscLZ streams alone. With the previous commit's check removed, fuzzed_frames_never_panic panics at the same subtraction. A 100x-iteration soak (different seed) found no other panic. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../clawhdf5-format/src/filters_bitshuffle.rs | 61 +++++++ crates/clawhdf5-format/src/filters_blosc.rs | 62 ++++++++ crates/clawhdf5-format/src/filters_bzip2.rs | 22 ++- crates/clawhdf5-format/src/filters_lzf.rs | 28 ++++ crates/clawhdf5-format/src/lib.rs | 10 ++ crates/clawhdf5-format/src/test_fuzz.rs | 149 ++++++++++++++++++ 6 files changed, 331 insertions(+), 1 deletion(-) create mode 100644 crates/clawhdf5-format/src/test_fuzz.rs diff --git a/crates/clawhdf5-format/src/filters_bitshuffle.rs b/crates/clawhdf5-format/src/filters_bitshuffle.rs index 83bab22..d20835e 100644 --- a/crates/clawhdf5-format/src/filters_bitshuffle.rs +++ b/crates/clawhdf5-format/src/filters_bitshuffle.rs @@ -374,4 +374,65 @@ mod tests { ctx.max_output = 100; assert!(bitshuffle_decode(&enc, &ctx).is_err()); } + + /// Random and mutated chunks, in every mode, and hostile `cd_values`: + /// errors are fine, panics are not. + #[cfg(feature = "bitshuffle")] + #[test] + fn fuzzed_chunks_never_panic() { + use crate::test_fuzz::{Rng, fuzz_decoder}; + let data: Vec = (0..3001u32) + .flat_map(|i| ((i / 7) as u16).to_le_bytes()) + .collect(); + for (comp, block) in [(0, 0), (0, 16), (2, 0), (2, 64), (3, 0), (3, 1024)] { + let f = ctx_for(vec![0, 4, 2, block, comp]); + let ctx = FilterContext { + filter: &f, + element_size: 2, + max_output: data.len(), + }; + let seeds = vec![ + bitshuffle_encode(&data, &ctx).unwrap(), + bitshuffle_encode(&data[..34], &ctx).unwrap(), + bitshuffle_encode(&data[..512], &ctx).unwrap(), + ]; + fuzz_decoder( + 0xb5 + comp as u64 * 7 + block as u64, + &seeds, + 4_000, + data.len(), + |s| bitshuffle_decode(s, &ctx), + ); + } + // Hostile filter parameters on a valid chunk. + let mut rng = Rng::new(0xcd); + let good = ctx_for(vec![0, 4, 2, 0, 2]); + let enc = bitshuffle_encode( + &data, + &FilterContext { + filter: &good, + element_size: 2, + max_output: data.len(), + }, + ) + .unwrap(); + for _ in 0..3_000 { + let cd: Vec = (0..rng.below(7)) + .map(|_| match rng.below(4) { + 0 => rng.below(5) as u32, + 1 => u32::MAX - rng.below(4) as u32, + 2 => 1 << rng.below(32), + _ => rng.next_u64() as u32, + }) + .collect(); + let f = ctx_for(cd); + let ctx = FilterContext { + filter: &f, + element_size: 2, + max_output: data.len(), + }; + let _ = bitshuffle_decode(&enc, &ctx); + let _ = bitshuffle_decode(&data, &ctx); + } + } } diff --git a/crates/clawhdf5-format/src/filters_blosc.rs b/crates/clawhdf5-format/src/filters_blosc.rs index 1bf1d15..625676b 100644 --- a/crates/clawhdf5-format/src/filters_blosc.rs +++ b/crates/clawhdf5-format/src/filters_blosc.rs @@ -622,4 +622,66 @@ mod tests { assert!(blosc_decompress(&frame, 1000).is_err(), "cbytes={cbytes}"); } } + + /// A BloscLZ frame (our encoder cannot write one): a single block, + /// one stream, no shuffle. + fn blosclz_frame() -> Vec { + let stream = [2, b'a', b'b', b'c', (6 << 5), 2, 0, b'Z']; + let mut f = vec![2u8, 1, 0, 1]; + for v in [12u32, 12, (HEADER + 4 + 4 + stream.len()) as u32] { + f.extend_from_slice(&v.to_le_bytes()); + } + f.extend_from_slice(&((HEADER + 4) as u32).to_le_bytes()); + f.extend_from_slice(&(stream.len() as u32).to_le_bytes()); + f.extend_from_slice(&stream); + f + } + + /// Random and mutated frames, every codec and shuffle: errors are fine, + /// panics are not. + #[test] + fn fuzzed_frames_never_panic() { + let limit = 6000; + let data: Vec = (0..1500u32).flat_map(|i| (i / 5).to_le_bytes()).collect(); + let mut seeds = vec![blosclz_frame()]; + for codec in [1u32, 3, 4, 5] { + for shuffle in [0u32, 1, 2] { + for (ts, n) in [(4usize, data.len()), (4, 520), (1, 300), (2, 4)] { + let f = desc(vec![2, 2, ts as u32, 0, 5, shuffle, codec]); + let ctx = FilterContext { + filter: &f, + element_size: ts, + max_output: n, + }; + seeds.push(blosc_encode(&data[..n], &ctx).unwrap()); + } + } + } + // Stored raw. + let f = desc(vec![2, 2, 4, 0, 0, 1, 1]); + let ctx = FilterContext { + filter: &f, + element_size: 4, + max_output: 64, + }; + seeds.push(blosc_encode(&data[..64], &ctx).unwrap()); + crate::test_fuzz::fuzz_decoder(0xb10, &seeds, 30_000, limit, |s| { + blosc_decompress(s, limit) + }); + } + + /// BloscLZ streams on their own, random and mutated. + #[test] + fn fuzzed_blosclz_streams_never_panic() { + let seed = blosclz_frame()[HEADER + 8..].to_vec(); + let mut out = [0u8; 64]; + crate::test_fuzz::fuzz_decoder(0xb11, &[seed], 30_000, 64, |s| { + let n = blosclz_decompress(s, &mut out); + if n == 0 { + Err(err("malformed")) + } else { + Ok(out[..n].to_vec()) + } + }); + } } diff --git a/crates/clawhdf5-format/src/filters_bzip2.rs b/crates/clawhdf5-format/src/filters_bzip2.rs index 2a78074..93e33a3 100644 --- a/crates/clawhdf5-format/src/filters_bzip2.rs +++ b/crates/clawhdf5-format/src/filters_bzip2.rs @@ -37,7 +37,10 @@ pub(crate) fn bzip2_decode(input: &[u8], ctx: &FilterContext<'_>) -> Result= input.len() @@ -109,4 +112,21 @@ mod tests { assert!(bzip2_decode(&enc, &small).is_err()); } } + + /// Random and mutated streams: errors are fine, panics are not. + #[test] + fn fuzzed_streams_never_panic() { + let f = desc(9); + let data: Vec = (0..4000u32).flat_map(|i| (i % 91).to_le_bytes()).collect(); + let ctx = FilterContext { + filter: &f, + element_size: 4, + max_output: data.len(), + }; + let seeds = vec![ + bzip2_encode(&data, &ctx).unwrap(), + bzip2_encode(&data[..40], &ctx).unwrap(), + ]; + crate::test_fuzz::fuzz_decoder(0xb2, &seeds, 3_000, data.len(), |s| bzip2_decode(s, &ctx)); + } } diff --git a/crates/clawhdf5-format/src/filters_lzf.rs b/crates/clawhdf5-format/src/filters_lzf.rs index 5657339..acb9763 100644 --- a/crates/clawhdf5-format/src/filters_lzf.rs +++ b/crates/clawhdf5-format/src/filters_lzf.rs @@ -229,4 +229,32 @@ mod tests { let c = lzf_compress(&[1u8; 100]); assert!(lzf_decompress(&c, 10, 99).is_err()); } + + /// Random and mutated streams: errors are fine, panics are not. + #[test] + fn fuzzed_streams_never_panic() { + let seeds: Vec> = [ + b"hello hello hello hello".to_vec(), + vec![7u8; 3000], + (0..2000u32).flat_map(|i| (i % 37).to_le_bytes()).collect(), + (0..500u32) + .map(|i| (i.wrapping_mul(2_654_435_761) >> 13) as u8) + .collect(), + ] + .iter() + .map(|d| lzf_compress(d)) + .collect(); + for limit in [0usize, 23, 4096, 8000] { + crate::test_fuzz::fuzz_decoder( + 0x1f2 + limit as u64, + &seeds[..1], + 5_000, + limit.max(23), + |s| lzf_decompress(s, limit, limit.max(23)), + ); + } + crate::test_fuzz::fuzz_decoder(0x1f3, &seeds, 20_000, 8000, |s| { + lzf_decompress(s, 8000, 8000) + }); + } } diff --git a/crates/clawhdf5-format/src/lib.rs b/crates/clawhdf5-format/src/lib.rs index 3e31686..905ce84 100644 --- a/crates/clawhdf5-format/src/lib.rs +++ b/crates/clawhdf5-format/src/lib.rs @@ -117,6 +117,16 @@ pub mod shared_message; pub mod signature; pub mod superblock; pub mod symbol_table; +#[cfg(all( + test, + any( + feature = "lzf", + feature = "bitshuffle", + feature = "bzip2", + feature = "blosc" + ) +))] +mod test_fuzz; pub mod type_builders; pub mod vds; pub mod vl_data; diff --git a/crates/clawhdf5-format/src/test_fuzz.rs b/crates/clawhdf5-format/src/test_fuzz.rs new file mode 100644 index 0000000..41515db --- /dev/null +++ b/crates/clawhdf5-format/src/test_fuzz.rs @@ -0,0 +1,149 @@ +//! Mutation fuzzing for the filter decoders (tests only). +//! +//! A decoder fed a random or mutated frame may fail, but must not panic — +//! tests build with overflow checks and debug assertions, so an unchecked +//! subtraction, multiplication or shift on a header field, or an +//! out-of-range slice, fails the test — and must not return more than its +//! output limit. + +#[cfg(not(feature = "std"))] +extern crate alloc; +#[cfg(not(feature = "std"))] +use alloc::vec::Vec; + +use crate::error::FormatError; + +/// xorshift64*: deterministic, so a failure reproduces. +pub(crate) struct Rng(u64); + +impl Rng { + pub(crate) fn new(seed: u64) -> Rng { + Rng(seed.max(1)) + } + + pub(crate) fn next_u64(&mut self) -> u64 { + let mut x = self.0; + x ^= x >> 12; + x ^= x << 25; + x ^= x >> 27; + self.0 = x; + x.wrapping_mul(0x2545_F491_4F6C_DD1D) + } + + /// Uniform in `0..n` (`n` > 0). + pub(crate) fn below(&mut self, n: usize) -> usize { + (self.next_u64() % n as u64) as usize + } + + pub(crate) fn bytes(&mut self, n: usize) -> Vec { + (0..n).map(|_| self.next_u64() as u8).collect() + } + + /// A u32 that tends to hit edge cases in size and offset fields. + fn interesting_u32(&mut self, len: usize) -> u32 { + match self.below(10) { + 0 => 0, + 1 => 1, + 2 => self.below(20) as u32, + 3 => 15 + self.below(3) as u32, + 4 => u32::MAX - self.below(16) as u32, + 5 => 1 << self.below(32), + 6 => (len as u32) + .wrapping_add(self.below(9) as u32) + .wrapping_sub(4), + 7 => i32::MAX as u32, + _ => self.next_u64() as u32, + } + } +} + +/// One to four random edits of `seed`. +pub(crate) fn mutate(rng: &mut Rng, seed: &[u8]) -> Vec { + let mut v = seed.to_vec(); + for _ in 0..1 + rng.below(4) { + let len = v.len(); + match rng.below(9) { + 0 if len > 0 => { + let i = rng.below(len); + v[i] ^= 1 << rng.below(8); + } + 1 if len > 0 => { + let i = rng.below(len); + v[i] = rng.next_u64() as u8; + } + 2 if len > 0 => { + let i = rng.below(len); + v[i] = [0, 0xff, 0x7f, 0x80, 0x20, 0x1f][rng.below(6)]; + } + // A size or offset field: little- or big-endian, anywhere, but + // most often in the first 32 bytes where headers live. + 3 | 4 if len >= 4 => { + let span = if rng.below(2) == 0 { len.min(32) } else { len }; + let i = rng.below(span - 3); + let x = rng.interesting_u32(len); + let b = if rng.below(2) == 0 { + x.to_le_bytes() + } else { + x.to_be_bytes() + }; + v[i..i + 4].copy_from_slice(&b); + } + 5 if len > 0 => v.truncate(rng.below(len)), + 6 => { + let n = 1 + rng.below(64); + let extra = rng.bytes(n); + v.extend_from_slice(&extra); + } + 7 if len > 1 => { + let a = rng.below(len); + let b = a + rng.below(len - a); + let copy = v[a..b].to_vec(); + let at = rng.below(len); + v.splice(at..at, copy); + } + _ if len > 0 => { + let i = rng.below(len); + v[i] = v[i].wrapping_add(1 + rng.below(3) as u8); + } + _ => v.push(rng.next_u64() as u8), + } + } + v +} + +/// Feed `iters` inputs to `decode`: mostly mutations of `seeds`, some pure +/// noise and some truncated seeds. Asserts only "no panic, output within +/// `limit`". +pub(crate) fn fuzz_decoder( + seed: u64, + seeds: &[Vec], + iters: usize, + limit: usize, + mut decode: impl FnMut(&[u8]) -> Result, FormatError>, +) { + assert!(!seeds.is_empty()); + let mut rng = Rng::new(seed); + for s in seeds { + // The seeds themselves must be valid, or the fuzz explores nothing. + decode(s).expect("seed frame must decode"); + } + for _ in 0..iters { + let input = match rng.below(16) { + 0 => { + let n = rng.below(96); + rng.bytes(n) + } + 1 => { + let s = &seeds[rng.below(seeds.len())]; + s[..rng.below(s.len() + 1)].to_vec() + } + _ => { + let s = &seeds[rng.below(seeds.len())]; + mutate(&mut rng, s) + } + }; + if let Ok(out) = decode(&input) { + assert!(out.len() <= limit, "decoded {} > limit {limit}", out.len()); + } + } +} From a5bd70216c26b787a2ddc51dd840c242d6cecaa6 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 01:22:26 -0500 Subject: [PATCH 11/13] fix(format): a chunk that decodes short is an error, not zero-filled HDF5 stores every chunk at the full chunk size (edge chunks are padded before filtering, and with "don't filter partial edge chunks" they are stored raw at full size), so a filter pipeline that decodes to fewer bytes means a corrupt chunk. Every chunk reader padded it with zeros and returned it as data. libhdf5 returns the rest uninitialised, or fails when the filter checks (Blosc with nbytes = 0). New filters::decompress_chunk_exact decodes and then requires exactly the chunk size, with the chunk's coordinates in the error (ChunkedReadError "chunk at [16] decoded to 16 bytes, expected 32"). It replaces decompress_chunk_masked at every chunk read path: the full read (sequential and lane-partitioned), the cached read, the sweep read, the planned-selection read, parallel_read's three decoders and partial_read's box read. decompress_chunk_masked is unchanged (fractal-heap huge objects already checked their own size). Blosc also rejects a frame declaring no data where the chunk size is known. Tests, each failing with the check disabled: filters and parallel_read unit tests; h5py_short_decoded_chunk_is_an_error (gzip chunks rewritten short with write_direct_chunk: 1-D, a 2-D edge chunk, and 40 chunks with shuffle, read through File full/cached/selection reads, a selection that avoids the chunk still reads, MmapFile and LazyFile, with and without the parallel feature); plugin_filters_interop short_decoding_chunks_are_errors (Blosc nbytes=0 and short, LZF and bzip2 short; the Blosc nbytes=0 case read as 16 zeros before). The existing don't-filter-partial-edge-chunks tests still pass. Conformance (tank, 2026-09-26): 573 of 697 ok, and no file changed class, reader result or first issue against the pre-fix run. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/chunked_read.rs | 14 ++-- crates/clawhdf5-format/src/filters.rs | 63 +++++++++++++++- crates/clawhdf5-format/src/filters_blosc.rs | 26 ++++++- crates/clawhdf5-format/src/parallel_read.rs | 68 ++++++++++++++++- crates/clawhdf5-format/src/partial_read.rs | 5 +- .../clawhdf5/tests/h5py_chunked_read_tests.rs | 74 +++++++++++++++++++ .../clawhdf5/tests/plugin_filters_interop.rs | 52 +++++++++++++ 7 files changed, 289 insertions(+), 13 deletions(-) diff --git a/crates/clawhdf5-format/src/chunked_read.rs b/crates/clawhdf5-format/src/chunked_read.rs index 3a3dc05..7e21447 100644 --- a/crates/clawhdf5-format/src/chunked_read.rs +++ b/crates/clawhdf5-format/src/chunked_read.rs @@ -15,7 +15,7 @@ use crate::datatype::Datatype; use crate::error::FormatError; use crate::extensible_array::{ExtensibleArrayHeader, read_extensible_array_chunks}; use crate::filter_pipeline::FilterPipeline; -use crate::filters::{all_filters_skipped, decompress_chunk_masked}; +use crate::filters::{all_filters_skipped, decompress_chunk_exact}; use crate::fixed_array::{FixedArrayHeader, read_fixed_array_chunks}; #[cfg(feature = "std")] use std::sync::Arc; @@ -65,12 +65,13 @@ fn decompress_all_chunks( let raw_chunk = &file_data[c_addr..c_addr + size]; let decompressed = if let Some(pl) = pipeline { - decompress_chunk_masked( + decompress_chunk_exact( raw_chunk, pl, chunk_total_bytes, element_size, chunk_info.filter_mask, + &chunk_info.offsets, )? } else { raw_chunk.to_vec() @@ -932,12 +933,13 @@ pub fn read_chunked_data_cached( let cache_them = total_bytes <= cache.max_bytes(); if let Some(pl) = pipeline { let decode = |c: &&ChunkInfo| -> Result, FormatError> { - decompress_chunk_masked( + decompress_chunk_exact( raw_bytes(c)?, pl, chunk_total_bytes, elem_size as u32, c.filter_mask, + &c.offsets, ) }; for batch in misses.chunks(DECODE_BATCH) { @@ -1217,12 +1219,13 @@ pub fn read_chunked_data_sweep( ensure_len(file_data, c_addr, size)?; let raw_chunk = &file_data[c_addr..c_addr + size]; let dec = if let Some(pl) = pipeline { - decompress_chunk_masked( + decompress_chunk_exact( raw_chunk, pl, chunk_total_bytes, elem_size as u32, chunk_info.filter_mask, + &coord, )? } else { raw_chunk.to_vec() @@ -1346,12 +1349,13 @@ pub fn read_chunked_data_indexed( ensure_len(file_data, c_addr, size)?; let raw_chunk = &file_data[c_addr..c_addr + size]; let decompressed = if let Some(pl) = pipeline { - decompress_chunk_masked( + decompress_chunk_exact( raw_chunk, pl, chunk_total_bytes, elem_size as u32, *filter_mask, + coord, )? } else { raw_chunk.to_vec() diff --git a/crates/clawhdf5-format/src/filters.rs b/crates/clawhdf5-format/src/filters.rs index aa8847c..99a5d8e 100644 --- a/crates/clawhdf5-format/src/filters.rs +++ b/crates/clawhdf5-format/src/filters.rs @@ -4,7 +4,7 @@ extern crate alloc; #[cfg(not(feature = "std"))] -use alloc::{boxed::Box, vec, vec::Vec}; +use alloc::{boxed::Box, format, vec, vec::Vec}; use crate::error::FormatError; #[cfg(feature = "deflate")] @@ -120,6 +120,34 @@ pub fn decompress_chunk_masked( Ok(data) } +/// Decode one stored chunk of a chunked dataset: [`decompress_chunk_masked`], +/// then require exactly `chunk_size` bytes (when `chunk_size` is known). +/// +/// HDF5 stores every chunk at the full chunk size — edge chunks are padded +/// before they are filtered, and an edge chunk left unfiltered is written +/// full-size too — so a pipeline that decodes to fewer bytes means a +/// corrupt chunk. libhdf5 fails such a read (or returns uninitialised +/// memory); it must never read back as zeros. `coords` (the chunk's offset +/// in the dataset) is named in the error. +pub fn decompress_chunk_exact( + compressed: &[u8], + pipeline: &FilterPipeline, + chunk_size: usize, + element_size: u32, + filter_mask: u32, + coords: &[u64], +) -> Result, FormatError> { + let data = + decompress_chunk_masked(compressed, pipeline, chunk_size, element_size, filter_mask)?; + if chunk_size != 0 && data.len() != chunk_size { + return Err(FormatError::ChunkedReadError(format!( + "chunk at {coords:?} decoded to {} bytes, expected {chunk_size}", + data.len() + ))); + } + Ok(data) +} + /// Apply a filter pipeline to compress a chunk. /// Filters are applied in FORWARD order for compression. pub fn compress_chunk( @@ -1516,6 +1544,39 @@ fn pcodec_decompress( #[cfg(test)] mod tests { + + /// A chunk whose pipeline decodes to fewer bytes than the chunk holds is + /// an error naming the chunk, never a short buffer the reader pads. + #[test] + fn short_decoded_chunk_is_an_error() { + let pipeline = FilterPipeline { + version: 2, + filters: vec![FilterDescription { + filter_id: FILTER_SHUFFLE, + name: None, + flags: 0, + client_data: vec![4], + }], + }; + let full = [7u8; 32]; + assert_eq!( + decompress_chunk_exact(&full, &pipeline, 32, 4, 0, &[8]).unwrap(), + full + ); + let err = decompress_chunk_exact(&full[..16], &pipeline, 32, 4, 0, &[8, 0]).unwrap_err(); + let msg = err.to_string(); + assert!( + msg.contains("[8, 0]") && msg.contains("16") && msg.contains("32"), + "{msg}" + ); + // Every filter skipped: the stored bytes are the chunk, still checked. + assert!(decompress_chunk_exact(&full[..16], &pipeline, 32, 4, 1, &[0]).is_err()); + // Unknown chunk size: not checked. + assert_eq!( + decompress_chunk_exact(&full[..16], &pipeline, 0, 4, 0, &[0]).unwrap(), + &full[..16] + ); + } use super::*; use crate::filter_pipeline::FilterDescription; diff --git a/crates/clawhdf5-format/src/filters_blosc.rs b/crates/clawhdf5-format/src/filters_blosc.rs index 625676b..3edc964 100644 --- a/crates/clawhdf5-format/src/filters_blosc.rs +++ b/crates/clawhdf5-format/src/filters_blosc.rs @@ -113,8 +113,15 @@ fn decode_stream( } /// Decode a Blosc-filtered chunk: one Blosc 1 frame. +/// +/// An HDF5 chunk is never empty, so a frame that decodes to nothing where +/// the chunk size is known is corrupt (libhdf5's filter fails it too). pub(crate) fn blosc_decode(input: &[u8], ctx: &FilterContext<'_>) -> Result, FormatError> { - blosc_decompress(input, ctx.output_limit()) + let out = blosc_decompress(input, ctx.output_limit())?; + if out.is_empty() && ctx.max_output != 0 { + return Err(err("empty frame for a non-empty chunk")); + } + Ok(out) } /// Decompress a Blosc 1 frame, refusing more than `limit` bytes of output. @@ -606,6 +613,23 @@ mod tests { assert!(blosc_encode(&data, &ctx0).is_err()); } + /// A frame that declares no data, for a chunk that has some. + #[test] + fn empty_frame_for_a_non_empty_chunk_is_an_error() { + let mut frame = vec![2u8, 1, 0x20, 4]; + for v in [0u32, 64, 16] { + frame.extend_from_slice(&v.to_le_bytes()); + } + assert_eq!(blosc_decompress(&frame, 64).unwrap(), b""); + let f = desc(vec![2, 2, 4, 64, 5, 1, 1]); + let ctx = FilterContext { + filter: &f, + element_size: 4, + max_output: 64, + }; + assert!(blosc_decode(&frame, &ctx).is_err()); + } + /// A frame whose header claims a compressed size smaller than the /// header itself, not stored raw: an error, not an arithmetic overflow /// (it panicked in debug builds). diff --git a/crates/clawhdf5-format/src/parallel_read.rs b/crates/clawhdf5-format/src/parallel_read.rs index 0bb2785..6593132 100644 --- a/crates/clawhdf5-format/src/parallel_read.rs +++ b/crates/clawhdf5-format/src/parallel_read.rs @@ -10,7 +10,7 @@ use crate::chunked_read::ChunkInfo; use crate::error::FormatError; use crate::filter_pipeline::FilterPipeline; -use crate::filters::decompress_chunk_masked; +use crate::filters::decompress_chunk_exact; use crate::lane_partition::{self, LaneStats, PartitionStats}; /// Threshold: only use parallel decompression when chunk count exceeds this. @@ -84,12 +84,13 @@ pub fn decompress_chunks_lane_partitioned( } let raw_chunk = &file_data[c_addr..c_addr + size]; - let decompressed = decompress_chunk_masked( + let decompressed = decompress_chunk_exact( raw_chunk, pipeline, chunk_total_bytes, element_size, chunk_info.filter_mask, + &chunk_info.offsets, )?; stats.chunks_processed += 1; @@ -160,12 +161,13 @@ pub fn decompress_chunks_parallel( } let raw_chunk = &file_data[c_addr..c_addr + size]; - let decompressed = decompress_chunk_masked( + let decompressed = decompress_chunk_exact( raw_chunk, pipeline, chunk_total_bytes, element_size, chunk_info.filter_mask, + &chunk_info.offsets, )?; Ok(DecompressedChunk { @@ -204,12 +206,13 @@ pub fn decompress_chunks_sequential( let raw_chunk = &file_data[c_addr..c_addr + size]; let decompressed = if let Some(pl) = pipeline { - decompress_chunk_masked( + decompress_chunk_exact( raw_chunk, pl, chunk_total_bytes, element_size, chunk_info.filter_mask, + &chunk_info.offsets, )? } else { raw_chunk.to_vec() @@ -218,3 +221,60 @@ pub fn decompress_chunks_sequential( } Ok(result) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::filter_pipeline::{FILTER_SHUFFLE, FilterDescription}; + + /// Eight shuffled 32-byte chunks; chunk 5 is stored short when `short`. + fn chunks(short: bool) -> (Vec, Vec) { + let mut file = Vec::new(); + let mut infos = Vec::new(); + for i in 0..8u64 { + let len = if short && i == 5 { 16 } else { 32 }; + infos.push(ChunkInfo { + chunk_size: len as u32, + filter_mask: 0, + offsets: vec![i * 8], + address: file.len() as u64, + }); + file.extend(core::iter::repeat_n(i as u8, len)); + } + (file, infos) + } + + /// Every parallel decoder refuses a chunk that decodes short, naming it. + #[test] + fn short_decoded_chunk_is_an_error() { + let pipeline = FilterPipeline { + version: 2, + filters: vec![FilterDescription { + filter_id: FILTER_SHUFFLE, + name: None, + flags: 0, + client_data: vec![4], + }], + }; + let (file, good) = chunks(false); + assert_eq!( + decompress_chunks_parallel(&file, &good, &pipeline, 32, 4).unwrap()[5], + [5u8; 32] + ); + let (file, bad) = chunks(true); + let errs = [ + decompress_chunks_lane_partitioned(&file, &bad, &pipeline, 32, 4, 1, Some(3)) + .map(|_| ()) + .unwrap_err(), + decompress_chunks_parallel(&file, &bad, &pipeline, 32, 4) + .map(|_| ()) + .unwrap_err(), + decompress_chunks_sequential(&file, &bad, Some(&pipeline), 32, 4) + .map(|_| ()) + .unwrap_err(), + ]; + for e in errs { + assert!(e.to_string().contains("[40]"), "{e}"); + } + } +} diff --git a/crates/clawhdf5-format/src/partial_read.rs b/crates/clawhdf5-format/src/partial_read.rs index 5cc3aaa..7d73599 100644 --- a/crates/clawhdf5-format/src/partial_read.rs +++ b/crates/clawhdf5-format/src/partial_read.rs @@ -22,7 +22,7 @@ use crate::data_read::extract_selection_from_buffer; use crate::dataspace::Dataspace; use crate::error::FormatError; use crate::filter_pipeline::FilterPipeline; -use crate::filters::{all_filters_skipped, decompress_chunk_masked}; +use crate::filters::{all_filters_skipped, decompress_chunk_exact}; use crate::selection::Selection; /// The smallest axis-aligned box containing every selected element, as @@ -330,12 +330,13 @@ pub fn read_selection( let decoded; let data: &[u8] = match pipeline { Some(pl) if !all_filters_skipped(pl, chunk.filter_mask) => { - decoded = decompress_chunk_masked( + decoded = decompress_chunk_exact( raw, pl, chunk_bytes, elem_size as u32, chunk.filter_mask, + &chunk.offsets[..rank], )?; &decoded } diff --git a/crates/clawhdf5/tests/h5py_chunked_read_tests.rs b/crates/clawhdf5/tests/h5py_chunked_read_tests.rs index 2b9e902..71fff7a 100644 --- a/crates/clawhdf5/tests/h5py_chunked_read_tests.rs +++ b/crates/clawhdf5/tests/h5py_chunked_read_tests.rs @@ -343,3 +343,77 @@ print("OK") let want: Vec = line[990..].iter().flat_map(|v| v.to_le_bytes()).collect(); assert_eq!(tail, want); } + +// --------------------------------------------------------------------------- +// Chunks that decode short +// --------------------------------------------------------------------------- + +/// HDF5 stores every chunk at the full chunk size, so a chunk whose filters +/// decode to fewer bytes is corrupt. libhdf5 returns the rest of such a +/// chunk uninitialised (or, for filters that check, fails); clawhdf5 padded +/// it with zeros and returned it as data. Every read path must fail, +/// naming the chunk, and chunks that decode fully must still read. +#[test] +fn h5py_short_decoded_chunk_is_an_error() { + skip_if_no_python!(); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("short.h5"); + let p = path.display().to_string(); + run_python(&format!( + r#" +import h5py, numpy as np, zlib +with h5py.File("{p}", "w") as f: + # 1-D, 4 gzip chunks of 8 i4; chunk 2 (offset 16) inflates to 16 bytes. + ds = f.create_dataset("line", shape=(32,), chunks=(8,), dtype=" = (1..=16i32).flat_map(i32::to_le_bytes).collect(); + assert_eq!(line.read_selection(&hyperslab(0, 16)).unwrap(), want); + let many = file.dataset("many").unwrap(); + assert!(many.read_selection(&hyperslab(190, 20)).is_err()); + + // The memory-mapped and lazy readers. + let mm = clawhdf5::MmapFile::open(&path).unwrap(); + assert!(mm.dataset("line").unwrap().read_i32().is_err()); + assert!(mm.dataset("many").unwrap().read_i32().is_err()); + let lazy = clawhdf5::LazyFile::open_mmap(&path).unwrap(); + assert!(lazy.dataset("line").unwrap().read_i32().is_err()); + assert!(lazy.dataset("grid").unwrap().read_f64().is_err()); +} diff --git a/crates/clawhdf5/tests/plugin_filters_interop.rs b/crates/clawhdf5/tests/plugin_filters_interop.rs index 647c007..81cc87c 100644 --- a/crates/clawhdf5/tests/plugin_filters_interop.rs +++ b/crates/clawhdf5/tests/plugin_filters_interop.rs @@ -415,3 +415,55 @@ with h5py.File(sys.argv[1], 'w') as f: ); } } + +/// A corrupt chunk must never read as zeros: a Blosc frame that declares no +/// data (libhdf5's filter fails it), and Blosc, LZF and bzip2 chunks that +/// decode short (libhdf5 returns the rest of the chunk uninitialised). +#[test] +fn short_decoding_chunks_are_errors() { + if !have_python("h5py, hdf5plugin") { + return; + } + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("short.h5"); + run_python( + r#" +import sys, struct, bz2 +import numpy as np, h5py, hdf5plugin +def lzf(b): + # One literal run per 32 bytes: a valid LZF stream. + return b"".join(bytes([len(b[i:i+32]) - 1]) + b[i:i+32] for i in range(0, len(b), 32)) +with h5py.File(sys.argv[1], 'w') as f: + kw = dict(shape=(16,), dtype=' = (0..16).collect(); + assert_eq!(file.dataset("lzf_ok").unwrap().read_i32().unwrap(), want); +} From 738b9491b2f8c00bffa002604e088bd6b367832f Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 01:23:58 -0500 Subject: [PATCH 12/13] fix(format): a codec can be registered for filter 32023 (Granular BitRound) With the pcodec feature, 32023 was a built-in entry (the legacy reader for the pcodec chunks clawhdf5 <= 2.7.0 wrote under that ID), so register_filter(32023, ...) was refused as "built in", although UnsupportedFilter(32023) names Granular BitRound as not implemented and the registry docs point to register_filter for such IDs. That entry is now shared: it claims only chunks whose filter is named "pcodec"; any other chunk with ID 32023 goes to the registered codec (or, with none registered, gets UnsupportedFilter as before), and writing 32023 uses the registered codec. Every other built-in ID still refuses registration. Test: a_codec_can_be_registered_for_granular_bitround (registers, round- trips chunks with no name and other names, still reads a legacy "pcodec" chunk with the built-in reader, and after unregistering reads nothing). It fails without the change ("filter 32023 ... is built in and cannot be re-registered"). It and the existing legacy-pcodec test share a lock, since the registry is process-wide. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/filter_registry.rs | 97 +++++++++++++++++-- crates/clawhdf5-format/src/filters.rs | 6 +- 2 files changed, 94 insertions(+), 9 deletions(-) diff --git a/crates/clawhdf5-format/src/filter_registry.rs b/crates/clawhdf5-format/src/filter_registry.rs index 62e22b8..6d39613 100644 --- a/crates/clawhdf5-format/src/filter_registry.rs +++ b/crates/clawhdf5-format/src/filter_registry.rs @@ -9,7 +9,11 @@ //! [`builtin_filters`] lists them. //! * **Registered filters** (`std` only) — codecs the application supplies //! for any other ID with [`register_filter`] (a [`FilterCodec`], or just a -//! decoding closure). A registered codec cannot shadow a built-in one. +//! decoding closure). A registered codec cannot shadow a built-in one, +//! except under 32023: that ID belongs to Granular BitRound, and the +//! built-in entry there only reads the pcodec chunks clawhdf5 <= 2.7.0 +//! wrote (filter name `"pcodec"`), so a codec registered for 32023 handles +//! every other chunk with that ID, and writes. //! //! An ID in neither tier fails with [`FormatError::UnsupportedFilter`], as it //! always has. @@ -118,6 +122,19 @@ impl BuiltinFilter { pub fn can_encode(&self) -> bool { self.encode.is_some() } + + /// Whether the built-in entry only borrows its ID for some chunks, so a + /// registered codec may take the rest: the legacy pcodec entry under + /// Granular BitRound's 32023, which claims only chunks named `"pcodec"`. + fn is_shared(&self) -> bool { + self.id == crate::filter_pipeline::FILTER_PCODEC_LEGACY + } + + /// Whether this entry decodes chunks written with `filter`. + fn claims(&self, filter: &crate::filter_pipeline::FilterDescription) -> bool { + !self.is_shared() + || filter.name.as_deref() == Some(crate::filter_pipeline::FILTER_PCODEC_LEGACY_NAME) + } } /// The filters compiled into this build, in ID order. @@ -198,7 +215,11 @@ mod custom { /// A plain closure `Fn(&[u8], &FilterContext) -> Result, FormatError>` /// registers a decoder. Replaces (and returns) an earlier registration for /// the same ID. Fails with [`FormatError::FilterError`] if `id` is a built-in -/// filter of this build: those cannot be overridden. +/// filter of this build: those cannot be overridden. The exception is 32023 +/// (Granular BitRound): with the `pcodec` feature the built-in entry there +/// reads only chunks whose filter is named `"pcodec"` (clawhdf5 <= 2.7.0's +/// files); a codec registered for 32023 decodes every other chunk with that +/// ID and does all the writing. #[cfg(feature = "std")] pub fn register_filter( id: u16, @@ -207,7 +228,7 @@ pub fn register_filter( where C: FilterCodec + 'static, { - if let Some(builtin) = builtin_filter(id) { + if let Some(builtin) = builtin_filter(id).filter(|b| !b.is_shared()) { return Err(FormatError::FilterError(format!( "filter {id} ({}) is built in and cannot be re-registered", builtin.name @@ -229,11 +250,13 @@ pub fn registered(id: u16) -> Option> { custom::with_read(|r| r.get(&id).cloned()) } -/// Undo filter `ctx.filter` on `input`: the built-in decoder if there is one, -/// else a registered one, else [`FormatError::UnsupportedFilter`]. +/// Undo filter `ctx.filter` on `input`: the built-in decoder if there is one +/// that claims the chunk, else a registered one, else the built-in decoder's +/// own refusal or [`FormatError::UnsupportedFilter`]. pub(crate) fn decode(input: &[u8], ctx: &FilterContext<'_>) -> Result, FormatError> { let id = ctx.filter.filter_id; - if let Some(builtin) = builtin_filter(id) { + let builtin = builtin_filter(id); + if let Some(builtin) = builtin.filter(|b| b.claims(ctx.filter)) { return (builtin.decode)(input, ctx); } #[cfg(feature = "std")] @@ -250,12 +273,21 @@ pub(crate) fn decode(input: &[u8], ctx: &FilterContext<'_>) -> Result, F } return Ok(out); } - Err(FormatError::UnsupportedFilter(id)) + match builtin { + Some(builtin) => (builtin.decode)(input, ctx), + None => Err(FormatError::UnsupportedFilter(id)), + } } /// Apply filter `ctx.filter` to `input`. pub(crate) fn encode(input: &[u8], ctx: &FilterContext<'_>) -> Result, FormatError> { let id = ctx.filter.filter_id; + #[cfg(feature = "std")] + if builtin_filter(id).is_some_and(|b| b.is_shared()) + && let Some(codec) = registered(id) + { + return codec.encode(input, ctx); + } if let Some(builtin) = builtin_filter(id) { return match builtin.encode { Some(encode) => encode(input, ctx), @@ -270,7 +302,7 @@ pub(crate) fn encode(input: &[u8], ctx: &FilterContext<'_>) -> Result, F } #[cfg(all(test, feature = "std"))] -mod tests { +pub(crate) mod tests { use super::*; use crate::filter_pipeline::{FILTER_FLETCHER32, FILTER_SHUFFLE, FilterPipeline}; use crate::filters::{compress_chunk, decompress_chunk}; @@ -368,6 +400,55 @@ mod tests { assert!(builtin_filter(FILTER_SHUFFLE).is_some()); } + /// Serialises the tests that register or read filter 32023 (the + /// registry is process-wide). + pub(crate) static ID_32023: std::sync::Mutex<()> = std::sync::Mutex::new(()); + + /// 32023 is Granular BitRound's ID; the `pcodec` build's built-in entry + /// there reads only clawhdf5 <= 2.7.0's pcodec chunks (named "pcodec"), + /// so a codec can be registered for the rest, and writes with it. + #[test] + fn a_codec_can_be_registered_for_granular_bitround() { + let _guard = ID_32023 + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let named = |name: Option<&str>| FilterPipeline { + version: 2, + filters: vec![FilterDescription { + filter_id: 32023, + name: name.map(Into::into), + flags: 0, + client_data: vec![7], + }], + }; + let prev = register_filter(32023, Xor).expect("32023 must be registrable"); + assert!(prev.is_none()); + let data = b"granular bitround".to_vec(); + for name in [None, Some("granular_bitround"), Some("test")] { + let pl = named(name); + let enc = compress_chunk(&data, &pl, 1).unwrap(); + assert_ne!(enc, data); + assert_eq!(decompress_chunk(&enc, &pl, data.len(), 1).unwrap(), data); + } + // clawhdf5 <= 2.7.0's pcodec chunks still go to the built-in reader. + #[cfg(feature = "pcodec")] + { + let raw: Vec = (0..64) + .flat_map(|i| (f64::from(i) * 0.5).to_le_bytes()) + .collect(); + let comp = crate::filters::pcodec_compress(&raw, 8).unwrap(); + let mut pl = named(Some("pcodec")); + pl.filters[0].client_data = vec![8]; + assert_eq!(decompress_chunk(&comp, &pl, raw.len(), 8).unwrap(), raw); + } + assert!(unregister_filter(32023)); + let pl = named(None); + assert!(matches!( + decompress_chunk(&data, &pl, data.len(), 1), + Err(FormatError::UnsupportedFilter(32023)) + )); + } + #[test] fn unsupported_filter_error_names_the_filter() { let msg = FormatError::UnsupportedFilter(32026).to_string(); diff --git a/crates/clawhdf5-format/src/filters.rs b/crates/clawhdf5-format/src/filters.rs index 99a5d8e..36f2e96 100644 --- a/crates/clawhdf5-format/src/filters.rs +++ b/crates/clawhdf5-format/src/filters.rs @@ -1446,7 +1446,7 @@ fn fletcher32_append(data: &[u8]) -> Result, FormatError> { // --------------------------------------------------------------------------- #[cfg(feature = "pcodec")] -fn pcodec_compress(data: &[u8], element_size: usize) -> Result, FormatError> { +pub(crate) fn pcodec_compress(data: &[u8], element_size: usize) -> Result, FormatError> { use pco::ChunkConfig; use pco::standalone::simple_compress; let config = ChunkConfig::default(); @@ -2758,6 +2758,10 @@ mod tests { #[cfg(feature = "pcodec")] fn pcodec_uses_private_id_and_reads_legacy_32023() { use crate::chunked_write::ChunkOptions; + #[cfg(feature = "std")] + let _guard = crate::filter_registry::tests::ID_32023 + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); let opts = ChunkOptions { pcodec: true, ..Default::default() From 17fc8b19645f0ac0c5c66cdfdb01286f019250b6 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 01:24:18 -0500 Subject: [PATCH 13/13] docs: changelog and known issues for the plugin-filter review fixes The short-decoding chunk (wrong data, pre-existing), the Blosc header underflow (crash) and filter 32023 registration, each with its date and what it changes; the conformance count is unchanged at 573 of 697. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 21 +++++++++++++++++++++ docs/known-issues.md | 11 ++++++++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9318dbc..796c459 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,27 @@ missing feature ("unsupported filter: 32026 (Blosc2, not implemented by clawhdf5)"). - **Not implemented:** Blosc2 (32026) and ZFP (32013) remain a clear error. +- **Wrong data: a chunk that decodes short read as zeros** (pre-existing, every + filter). HDF5 stores every chunk at the full chunk size, so a filter + pipeline that decodes to fewer bytes means a corrupt chunk; every chunk + reader (full, cached, selection, parallel, partial) padded it with zeros. + It is now an error naming the chunk ("chunk at [16] decoded to 16 bytes, + expected 32"), via the new `filters::decompress_chunk_exact`. libhdf5 + returns the rest of such a chunk uninitialised, or fails when the filter + checks. A Blosc frame declaring no data for a non-empty chunk is an error + too. Legitimate edge chunks are unaffected (they are stored full-size, + filtered or not); conformance is unchanged at 573 of 697, with no file + changing class. +- **Crash: a hostile Blosc chunk panicked** in builds with overflow checks + (debug builds, `cargo test`, `maturin develop`): a frame size below the + 16-byte header underflowed. It is now an error. Every new decoder (LZF, + bitshuffle, bzip2, Blosc/BloscLZ) is fuzzed with random and mutated frames + in the unit tests. +- **`register_filter(32023, ..)` works with the `pcodec` feature.** 32023 is + Granular BitRound's ID; the built-in entry there only reads clawhdf5 + <= 2.7.0's pcodec chunks (filter name `"pcodec"`), so a registered codec now + handles every other chunk with that ID, and writes. It was refused as + "built in". ### Upgrade Notes - **HDF5 correctness audit (2026-09-25).** A sweep of 686 public files (the diff --git a/docs/known-issues.md b/docs/known-issues.md index 53c4996..11c2ea9 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -140,7 +140,16 @@ fill-value item that did is fixed). 697 ok). **Still open:** Blosc2 (32026 — hdf5plugin stores each chunk as a Blosc2 super-chunk frame, and n-D chunks as B2ND arrays) and ZFP (32013); both fail with an `UnsupportedFilter` error that names the filter, and - either can be plugged in with `filter_registry::register_filter`. + either can be plugged in with `filter_registry::register_filter` (32023, + Granular BitRound, too, since 2026-09-26 even with the `pcodec` feature). +- **Wrong data: a chunk whose filters decode to fewer bytes than the chunk + read with zeros for the missing bytes** (any filter; found reviewing the plugin + filters). **Fixed 2026-09-26:** it is an error naming the chunk. A corrupt + chunk must never read as zeros. Unfiltered chunks are read at their stored + size and are not checked this way. +- **Crash:** a hostile Blosc chunk (frame size below its header) panicked in + builds with overflow checks. **Fixed 2026-09-26**; the new decoders are + fuzzed in the unit tests. - **Header checks:** on 12 CVE datasets libhdf5 rejects a corrupt header and we read data anyway. We need stricter header checks. - **Writer:**