Merge branch 'feat/p2b-blosc2' into feat/p2b-scale

# Conflicts:
#	CHANGELOG.md
This commit is contained in:
osobh
2026-09-26 11:57:11 -05:00
60 changed files with 2276 additions and 33 deletions
@@ -5,7 +5,8 @@
//! * **Built-in filters** — a static table of the filters compiled into this
//! build: the HDF5 standard filters (deflate, shuffle, Fletcher32, szip,
//! N-Bit, scale-offset) and the plugin filters whose cargo features are
//! enabled (LZ4, Zstandard, pcodec, LZF, bitshuffle, bzip2, blosc).
//! enabled (LZ4, Zstandard, pcodec, LZF, bitshuffle, bzip2, blosc,
//! blosc2).
//! [`builtin_filters`] lists them.
//! * **Registered filters** (`std` only) — codecs the application supplies
//! for any other ID with [`register_filter`] (a [`FilterCodec`], or just a
@@ -166,7 +167,7 @@ pub fn known_filter(id: u16) -> Option<(&'static str, Option<&'static str>)> {
32019 => ("JPEG", None),
32022 => ("BitGroom", None),
32023 => ("Granular BitRound", None),
32026 => ("Blosc2", None),
32026 => ("Blosc2", Some("blosc2")),
_ => return None,
})
}
@@ -452,12 +453,12 @@ pub(crate) mod tests {
#[test]
fn unsupported_filter_error_names_the_filter() {
let msg = FormatError::UnsupportedFilter(32026).to_string();
assert!(msg.contains("Blosc2") && msg.contains("`blosc2`"), "{msg}");
let msg = FormatError::UnsupportedFilter(32013).to_string();
assert!(
msg.contains("Blosc2") && msg.contains("not implemented"),
msg.contains("ZFP") && msg.contains("not implemented"),
"{msg}"
);
let msg = FormatError::UnsupportedFilter(32013).to_string();
assert!(msg.contains("ZFP"), "{msg}");
let msg = FormatError::UnsupportedFilter(32000).to_string();
assert!(msg.contains("LZF") && msg.contains("`lzf`"), "{msg}");
assert_eq!(
+7
View File
@@ -455,6 +455,13 @@ pub(crate) static BUILTIN_FILTERS: &[BuiltinFilter] = &[
},
encode: None,
},
#[cfg(feature = "blosc2")]
BuiltinFilter {
id: crate::filter_pipeline::FILTER_BLOSC2,
name: "blosc2",
decode: crate::filters_blosc2::blosc2_decode,
encode: None,
},
];
/// Decode the HDF5 scale-offset filter (id 6).
@@ -218,12 +218,20 @@ pub(crate) fn bitshuffle_decode(
}
/// Decode Zstandard frames into exactly `dst`, failing if they hold more.
///
/// ruzstd reserves a frame's declared window (by default up to 100 MiB)
/// before decoding it, so the window is capped at what the output could
/// need: twice `dst` (window sizes are rounded up), and at least 128 KiB.
/// The encoders behind these filters (c-blosc, c-blosc2, bitshuffle)
/// compress each block in one call with its size known, so libzstd's
/// window never exceeds the block.
#[cfg(any(feature = "bitshuffle", feature = "blosc"))]
pub(crate) fn zstd_decode_into(
decoder: &mut ruzstd::decoding::FrameDecoder,
frames: &[u8],
dst: &mut [u8],
) -> Result<usize, FormatError> {
decoder.set_max_window_size((2 * dst.len()).max(1 << 17) as u64);
decoder
.decode_all(frames, dst)
.map_err(|e| FormatError::DecompressionError(format!("zstd: {e}")))
+3 -3
View File
@@ -49,7 +49,7 @@ fn le32(b: &[u8], at: usize) -> Result<usize, FormatError> {
/// The codec inside a Blosc frame (flags bits 5-7).
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum Codec {
pub(crate) enum Codec {
BloscLz,
Lz4,
Snappy,
@@ -58,7 +58,7 @@ enum Codec {
}
impl Codec {
fn from_flags(flags: u8) -> Result<Codec, FormatError> {
pub(crate) fn from_flags(flags: u8) -> Result<Codec, FormatError> {
match flags >> 5 {
0 => Ok(Codec::BloscLz),
1 => Ok(Codec::Lz4),
@@ -71,7 +71,7 @@ impl Codec {
}
/// Decode one codec stream into exactly `dst`.
fn decode_stream(
pub(crate) fn decode_stream(
codec: Codec,
src: &[u8],
dst: &mut [u8],
File diff suppressed because it is too large Load Diff
+2
View File
@@ -87,6 +87,8 @@ pub mod filters;
mod filters_bitshuffle;
#[cfg(feature = "blosc")]
pub mod filters_blosc;
#[cfg(feature = "blosc2")]
pub mod filters_blosc2;
#[cfg(feature = "bzip2")]
mod filters_bzip2;
#[cfg(feature = "lzf")]