Merge branch 'feat/p2b-blosc2' into feat/p2b-scale
# Conflicts: # CHANGELOG.md
This commit is contained in:
@@ -5,7 +5,8 @@
|
||||
//! * **Built-in filters** — a static table of the filters compiled into this
|
||||
//! build: the HDF5 standard filters (deflate, shuffle, Fletcher32, szip,
|
||||
//! N-Bit, scale-offset) and the plugin filters whose cargo features are
|
||||
//! enabled (LZ4, Zstandard, pcodec, LZF, bitshuffle, bzip2, blosc).
|
||||
//! enabled (LZ4, Zstandard, pcodec, LZF, bitshuffle, bzip2, blosc,
|
||||
//! blosc2).
|
||||
//! [`builtin_filters`] lists them.
|
||||
//! * **Registered filters** (`std` only) — codecs the application supplies
|
||||
//! for any other ID with [`register_filter`] (a [`FilterCodec`], or just a
|
||||
@@ -166,7 +167,7 @@ pub fn known_filter(id: u16) -> Option<(&'static str, Option<&'static str>)> {
|
||||
32019 => ("JPEG", None),
|
||||
32022 => ("BitGroom", None),
|
||||
32023 => ("Granular BitRound", None),
|
||||
32026 => ("Blosc2", None),
|
||||
32026 => ("Blosc2", Some("blosc2")),
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
@@ -452,12 +453,12 @@ pub(crate) mod tests {
|
||||
#[test]
|
||||
fn unsupported_filter_error_names_the_filter() {
|
||||
let msg = FormatError::UnsupportedFilter(32026).to_string();
|
||||
assert!(msg.contains("Blosc2") && msg.contains("`blosc2`"), "{msg}");
|
||||
let msg = FormatError::UnsupportedFilter(32013).to_string();
|
||||
assert!(
|
||||
msg.contains("Blosc2") && msg.contains("not implemented"),
|
||||
msg.contains("ZFP") && msg.contains("not implemented"),
|
||||
"{msg}"
|
||||
);
|
||||
let msg = FormatError::UnsupportedFilter(32013).to_string();
|
||||
assert!(msg.contains("ZFP"), "{msg}");
|
||||
let msg = FormatError::UnsupportedFilter(32000).to_string();
|
||||
assert!(msg.contains("LZF") && msg.contains("`lzf`"), "{msg}");
|
||||
assert_eq!(
|
||||
|
||||
@@ -455,6 +455,13 @@ pub(crate) static BUILTIN_FILTERS: &[BuiltinFilter] = &[
|
||||
},
|
||||
encode: None,
|
||||
},
|
||||
#[cfg(feature = "blosc2")]
|
||||
BuiltinFilter {
|
||||
id: crate::filter_pipeline::FILTER_BLOSC2,
|
||||
name: "blosc2",
|
||||
decode: crate::filters_blosc2::blosc2_decode,
|
||||
encode: None,
|
||||
},
|
||||
];
|
||||
|
||||
/// Decode the HDF5 scale-offset filter (id 6).
|
||||
|
||||
@@ -218,12 +218,20 @@ pub(crate) fn bitshuffle_decode(
|
||||
}
|
||||
|
||||
/// Decode Zstandard frames into exactly `dst`, failing if they hold more.
|
||||
///
|
||||
/// ruzstd reserves a frame's declared window (by default up to 100 MiB)
|
||||
/// before decoding it, so the window is capped at what the output could
|
||||
/// need: twice `dst` (window sizes are rounded up), and at least 128 KiB.
|
||||
/// The encoders behind these filters (c-blosc, c-blosc2, bitshuffle)
|
||||
/// compress each block in one call with its size known, so libzstd's
|
||||
/// window never exceeds the block.
|
||||
#[cfg(any(feature = "bitshuffle", feature = "blosc"))]
|
||||
pub(crate) fn zstd_decode_into(
|
||||
decoder: &mut ruzstd::decoding::FrameDecoder,
|
||||
frames: &[u8],
|
||||
dst: &mut [u8],
|
||||
) -> Result<usize, FormatError> {
|
||||
decoder.set_max_window_size((2 * dst.len()).max(1 << 17) as u64);
|
||||
decoder
|
||||
.decode_all(frames, dst)
|
||||
.map_err(|e| FormatError::DecompressionError(format!("zstd: {e}")))
|
||||
|
||||
@@ -49,7 +49,7 @@ fn le32(b: &[u8], at: usize) -> Result<usize, FormatError> {
|
||||
|
||||
/// The codec inside a Blosc frame (flags bits 5-7).
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum Codec {
|
||||
pub(crate) enum Codec {
|
||||
BloscLz,
|
||||
Lz4,
|
||||
Snappy,
|
||||
@@ -58,7 +58,7 @@ enum Codec {
|
||||
}
|
||||
|
||||
impl Codec {
|
||||
fn from_flags(flags: u8) -> Result<Codec, FormatError> {
|
||||
pub(crate) fn from_flags(flags: u8) -> Result<Codec, FormatError> {
|
||||
match flags >> 5 {
|
||||
0 => Ok(Codec::BloscLz),
|
||||
1 => Ok(Codec::Lz4),
|
||||
@@ -71,7 +71,7 @@ impl Codec {
|
||||
}
|
||||
|
||||
/// Decode one codec stream into exactly `dst`.
|
||||
fn decode_stream(
|
||||
pub(crate) fn decode_stream(
|
||||
codec: Codec,
|
||||
src: &[u8],
|
||||
dst: &mut [u8],
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -87,6 +87,8 @@ pub mod filters;
|
||||
mod filters_bitshuffle;
|
||||
#[cfg(feature = "blosc")]
|
||||
pub mod filters_blosc;
|
||||
#[cfg(feature = "blosc2")]
|
||||
pub mod filters_blosc2;
|
||||
#[cfg(feature = "bzip2")]
|
||||
mod filters_bzip2;
|
||||
#[cfg(feature = "lzf")]
|
||||
|
||||
Reference in New Issue
Block a user