feat(format): a filter registry — filters are looked up by ID
decompress_chunk_masked and compress_chunk matched on the filter ID. They now look the ID up in filter_registry: a static table of the built-in filters compiled into this build (a filter whose cargo feature is off is simply absent), then the codecs an application registered at run time with register_filter (a FilterCodec, or a plain decoding closure). Registered codecs cannot shadow a built-in one, and their output is held to the same per-stage bound as the built-in decoders. An ID in neither tier still fails with UnsupportedFilter(id). The "feature off" stub functions that returned UnsupportedFilter are gone: the table leaves those filters out instead. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
@@ -0,0 +1,355 @@
|
||||
//! Filter registry: every filter is looked up here by its HDF5 filter ID.
|
||||
//!
|
||||
//! Two tiers:
|
||||
//!
|
||||
//! * **Built-in filters** — a static table of the filters compiled into this
|
||||
//! build: the HDF5 standard filters (deflate, shuffle, Fletcher32, szip,
|
||||
//! N-Bit, scale-offset) and the plugin filters whose cargo features are
|
||||
//! enabled (LZ4, Zstandard, pcodec, LZF, bitshuffle, bzip2, blosc).
|
||||
//! [`builtin_filters`] lists them.
|
||||
//! * **Registered filters** (`std` only) — codecs the application supplies
|
||||
//! for any other ID with [`register_filter`] (a [`FilterCodec`], or just a
|
||||
//! decoding closure). A registered codec cannot shadow a built-in one.
|
||||
//!
|
||||
//! An ID in neither tier fails with [`FormatError::UnsupportedFilter`], as it
|
||||
//! always has.
|
||||
//!
|
||||
//! ```
|
||||
//! # #[cfg(feature = "std")] {
|
||||
//! use clawhdf5_format::filter_registry::{self, FilterContext};
|
||||
//! use clawhdf5_format::error::FormatError;
|
||||
//!
|
||||
//! // A toy filter in the private-use range: every byte XORed with 0x5A.
|
||||
//! filter_registry::register_filter(300, |input: &[u8], _ctx: &FilterContext<'_>| {
|
||||
//! Ok::<_, FormatError>(input.iter().map(|b| b ^ 0x5A).collect())
|
||||
//! })
|
||||
//! .unwrap();
|
||||
//! assert!(filter_registry::is_filter_available(300));
|
||||
//! filter_registry::unregister_filter(300);
|
||||
//! # }
|
||||
//! ```
|
||||
|
||||
#[cfg(not(feature = "std"))]
|
||||
extern crate alloc;
|
||||
|
||||
#[cfg(not(feature = "std"))]
|
||||
use alloc::vec::Vec;
|
||||
|
||||
use crate::error::FormatError;
|
||||
use crate::filter_pipeline::FilterDescription;
|
||||
|
||||
/// What a codec is told about the filter it is applying.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct FilterContext<'a> {
|
||||
/// The filter as recorded in the dataset's filter pipeline: its ID, name,
|
||||
/// flags and client data (`cd_values`).
|
||||
pub filter: &'a FilterDescription,
|
||||
/// Size in bytes of one dataset element (the datatype's size).
|
||||
pub element_size: usize,
|
||||
/// Decoding only: the most bytes this stage may produce — what entered
|
||||
/// the filter when the chunk was written. 0 means unknown; a decoder then
|
||||
/// falls back to a fixed ceiling. Always 0 when encoding.
|
||||
pub max_output: usize,
|
||||
}
|
||||
|
||||
impl FilterContext<'_> {
|
||||
/// The filter's client data (`cd_values`).
|
||||
pub fn client_data(&self) -> &[u32] {
|
||||
&self.filter.client_data
|
||||
}
|
||||
|
||||
/// The largest output a decoder should allow: [`Self::max_output`], or
|
||||
/// 256 MiB when that is unknown.
|
||||
pub fn output_limit(&self) -> usize {
|
||||
if self.max_output != 0 {
|
||||
self.max_output
|
||||
} else {
|
||||
crate::filters::MAX_DECOMPRESS_SIZE
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A filter implementation.
|
||||
///
|
||||
/// `decode` undoes the filter (the read direction). `encode` applies it (the
|
||||
/// write direction); the default refuses with
|
||||
/// [`FormatError::UnsupportedFilter`], which is right for a read-only codec.
|
||||
pub trait FilterCodec: Send + Sync {
|
||||
/// Undo the filter on one chunk. The output must not exceed
|
||||
/// [`FilterContext::output_limit`]; the pipeline rejects a larger one.
|
||||
fn decode(&self, input: &[u8], ctx: &FilterContext<'_>) -> Result<Vec<u8>, FormatError>;
|
||||
|
||||
/// Apply the filter to one chunk.
|
||||
fn encode(&self, input: &[u8], ctx: &FilterContext<'_>) -> Result<Vec<u8>, FormatError> {
|
||||
let _ = input;
|
||||
Err(FormatError::UnsupportedFilter(ctx.filter.filter_id))
|
||||
}
|
||||
}
|
||||
|
||||
/// Any `Fn(&[u8], &FilterContext) -> Result<Vec<u8>, FormatError>` is a
|
||||
/// decode-only codec.
|
||||
impl<F> FilterCodec for F
|
||||
where
|
||||
F: Fn(&[u8], &FilterContext<'_>) -> Result<Vec<u8>, FormatError> + Send + Sync,
|
||||
{
|
||||
fn decode(&self, input: &[u8], ctx: &FilterContext<'_>) -> Result<Vec<u8>, FormatError> {
|
||||
self(input, ctx)
|
||||
}
|
||||
}
|
||||
|
||||
/// Signature of a built-in filter's decoder or encoder.
|
||||
pub type BuiltinFn = fn(&[u8], &FilterContext<'_>) -> Result<Vec<u8>, FormatError>;
|
||||
|
||||
/// A filter compiled into this build.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct BuiltinFilter {
|
||||
/// HDF5 filter ID.
|
||||
pub id: u16,
|
||||
/// Human-readable name.
|
||||
pub name: &'static str,
|
||||
/// Decoder.
|
||||
pub(crate) decode: BuiltinFn,
|
||||
/// Encoder, if this build can write the filter.
|
||||
pub(crate) encode: Option<BuiltinFn>,
|
||||
}
|
||||
|
||||
impl BuiltinFilter {
|
||||
/// Whether this build can write the filter as well as read it.
|
||||
pub fn can_encode(&self) -> bool {
|
||||
self.encode.is_some()
|
||||
}
|
||||
}
|
||||
|
||||
/// The filters compiled into this build, in ID order.
|
||||
pub fn builtin_filters() -> &'static [BuiltinFilter] {
|
||||
crate::filters::BUILTIN_FILTERS
|
||||
}
|
||||
|
||||
/// The built-in filter with this ID, if it is compiled in.
|
||||
pub fn builtin_filter(id: u16) -> Option<&'static BuiltinFilter> {
|
||||
builtin_filters().iter().find(|f| f.id == id)
|
||||
}
|
||||
|
||||
/// Whether a chunk filtered with `id` can be decoded: a built-in filter or a
|
||||
/// registered one.
|
||||
pub fn is_filter_available(id: u16) -> bool {
|
||||
if builtin_filter(id).is_some() {
|
||||
return true;
|
||||
}
|
||||
#[cfg(feature = "std")]
|
||||
{
|
||||
registered(id).is_some()
|
||||
}
|
||||
#[cfg(not(feature = "std"))]
|
||||
{
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "std")]
|
||||
mod custom {
|
||||
use super::FilterCodec;
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::{Arc, PoisonError, RwLock};
|
||||
|
||||
pub(super) type Registry = BTreeMap<u16, Arc<dyn FilterCodec>>;
|
||||
|
||||
static REGISTRY: RwLock<Registry> = RwLock::new(BTreeMap::new());
|
||||
|
||||
pub(super) fn with_read<R>(f: impl FnOnce(&Registry) -> R) -> R {
|
||||
// A panic while holding the lock cannot leave the map half-updated
|
||||
// (every update is a single insert/remove), so poisoning is ignored.
|
||||
f(®ISTRY.read().unwrap_or_else(PoisonError::into_inner))
|
||||
}
|
||||
|
||||
pub(super) fn with_write<R>(f: impl FnOnce(&mut Registry) -> R) -> R {
|
||||
f(&mut REGISTRY.write().unwrap_or_else(PoisonError::into_inner))
|
||||
}
|
||||
}
|
||||
|
||||
/// Register a codec for filter `id`, process-wide. It is used for every
|
||||
/// chunk read (and, if it implements [`FilterCodec::encode`], written) with
|
||||
/// that filter ID, by every file.
|
||||
///
|
||||
/// A plain closure `Fn(&[u8], &FilterContext) -> Result<Vec<u8>, FormatError>`
|
||||
/// registers a decoder. Replaces (and returns) an earlier registration for
|
||||
/// the same ID. Fails with [`FormatError::FilterError`] if `id` is a built-in
|
||||
/// filter of this build: those cannot be overridden.
|
||||
#[cfg(feature = "std")]
|
||||
pub fn register_filter<C>(
|
||||
id: u16,
|
||||
codec: C,
|
||||
) -> Result<Option<std::sync::Arc<dyn FilterCodec>>, FormatError>
|
||||
where
|
||||
C: FilterCodec + 'static,
|
||||
{
|
||||
if let Some(builtin) = builtin_filter(id) {
|
||||
return Err(FormatError::FilterError(format!(
|
||||
"filter {id} ({}) is built in and cannot be re-registered",
|
||||
builtin.name
|
||||
)));
|
||||
}
|
||||
let codec: std::sync::Arc<dyn FilterCodec> = std::sync::Arc::new(codec);
|
||||
Ok(custom::with_write(|r| r.insert(id, codec)))
|
||||
}
|
||||
|
||||
/// Remove the codec registered for `id`. Returns whether one was registered.
|
||||
#[cfg(feature = "std")]
|
||||
pub fn unregister_filter(id: u16) -> bool {
|
||||
custom::with_write(|r| r.remove(&id).is_some())
|
||||
}
|
||||
|
||||
/// The codec registered for `id`, if any.
|
||||
#[cfg(feature = "std")]
|
||||
pub fn registered(id: u16) -> Option<std::sync::Arc<dyn FilterCodec>> {
|
||||
custom::with_read(|r| r.get(&id).cloned())
|
||||
}
|
||||
|
||||
/// Undo filter `ctx.filter` on `input`: the built-in decoder if there is one,
|
||||
/// else a registered one, else [`FormatError::UnsupportedFilter`].
|
||||
pub(crate) fn decode(input: &[u8], ctx: &FilterContext<'_>) -> Result<Vec<u8>, FormatError> {
|
||||
let id = ctx.filter.filter_id;
|
||||
if let Some(builtin) = builtin_filter(id) {
|
||||
return (builtin.decode)(input, ctx);
|
||||
}
|
||||
#[cfg(feature = "std")]
|
||||
if let Some(codec) = registered(id) {
|
||||
let out = codec.decode(input, ctx)?;
|
||||
// A registered codec is outside our control: hold it to the same
|
||||
// bound the built-in decoders enforce.
|
||||
if out.len() > ctx.output_limit() {
|
||||
return Err(FormatError::DecompressionError(format!(
|
||||
"filter {id}: decoded {} bytes, more than the {} the chunk can hold",
|
||||
out.len(),
|
||||
ctx.output_limit()
|
||||
)));
|
||||
}
|
||||
return Ok(out);
|
||||
}
|
||||
Err(FormatError::UnsupportedFilter(id))
|
||||
}
|
||||
|
||||
/// Apply filter `ctx.filter` to `input`.
|
||||
pub(crate) fn encode(input: &[u8], ctx: &FilterContext<'_>) -> Result<Vec<u8>, FormatError> {
|
||||
let id = ctx.filter.filter_id;
|
||||
if let Some(builtin) = builtin_filter(id) {
|
||||
return match builtin.encode {
|
||||
Some(encode) => encode(input, ctx),
|
||||
None => Err(FormatError::UnsupportedFilter(id)),
|
||||
};
|
||||
}
|
||||
#[cfg(feature = "std")]
|
||||
if let Some(codec) = registered(id) {
|
||||
return codec.encode(input, ctx);
|
||||
}
|
||||
Err(FormatError::UnsupportedFilter(id))
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "std"))]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::filter_pipeline::{FILTER_FLETCHER32, FILTER_SHUFFLE, FilterPipeline};
|
||||
use crate::filters::{compress_chunk, decompress_chunk};
|
||||
|
||||
fn pipeline(id: u16) -> FilterPipeline {
|
||||
FilterPipeline {
|
||||
version: 2,
|
||||
filters: vec![FilterDescription {
|
||||
filter_id: id,
|
||||
name: Some("test".into()),
|
||||
flags: 0,
|
||||
client_data: vec![7],
|
||||
}],
|
||||
}
|
||||
}
|
||||
|
||||
struct Xor;
|
||||
impl FilterCodec for Xor {
|
||||
fn decode(&self, input: &[u8], ctx: &FilterContext<'_>) -> Result<Vec<u8>, FormatError> {
|
||||
let k = ctx.client_data()[0] as u8;
|
||||
Ok(input.iter().map(|b| b ^ k).collect())
|
||||
}
|
||||
fn encode(&self, input: &[u8], ctx: &FilterContext<'_>) -> Result<Vec<u8>, FormatError> {
|
||||
self.decode(input, ctx)
|
||||
}
|
||||
}
|
||||
|
||||
// Each test uses its own ID: the registry is process-wide and tests run
|
||||
// in parallel.
|
||||
|
||||
#[test]
|
||||
fn unknown_filter_keeps_its_error() {
|
||||
let err = decompress_chunk(b"abc", &pipeline(311), 3, 1).unwrap_err();
|
||||
assert_eq!(err, FormatError::UnsupportedFilter(311));
|
||||
let err = compress_chunk(b"abc", &pipeline(311), 1).unwrap_err();
|
||||
assert_eq!(err, FormatError::UnsupportedFilter(311));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn registered_codec_round_trips_through_the_pipeline() {
|
||||
assert!(!is_filter_available(312));
|
||||
assert!(register_filter(312, Xor).unwrap().is_none());
|
||||
assert!(is_filter_available(312));
|
||||
let data = b"hello, registry".to_vec();
|
||||
let enc = compress_chunk(&data, &pipeline(312), 1).unwrap();
|
||||
assert_ne!(enc, data);
|
||||
assert_eq!(
|
||||
decompress_chunk(&enc, &pipeline(312), data.len(), 1).unwrap(),
|
||||
data
|
||||
);
|
||||
assert!(unregister_filter(312));
|
||||
assert!(!unregister_filter(312));
|
||||
assert_eq!(
|
||||
decompress_chunk(&enc, &pipeline(312), data.len(), 1).unwrap_err(),
|
||||
FormatError::UnsupportedFilter(312)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn closure_registers_a_decoder_only() {
|
||||
register_filter(313, |input: &[u8], _ctx: &FilterContext<'_>| {
|
||||
Ok(input.iter().rev().copied().collect())
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
decompress_chunk(b"abc", &pipeline(313), 3, 1).unwrap(),
|
||||
b"cba"
|
||||
);
|
||||
assert_eq!(
|
||||
compress_chunk(b"abc", &pipeline(313), 1).unwrap_err(),
|
||||
FormatError::UnsupportedFilter(313)
|
||||
);
|
||||
unregister_filter(313);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn registered_decoder_output_is_bounded() {
|
||||
register_filter(314, |_input: &[u8], _ctx: &FilterContext<'_>| {
|
||||
Ok(vec![0u8; 1000])
|
||||
})
|
||||
.unwrap();
|
||||
let err = decompress_chunk(b"abc", &pipeline(314), 10, 1).unwrap_err();
|
||||
assert!(matches!(err, FormatError::DecompressionError(_)), "{err:?}");
|
||||
unregister_filter(314);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn builtins_cannot_be_overridden() {
|
||||
for id in [FILTER_SHUFFLE, FILTER_FLETCHER32] {
|
||||
let Err(err) = register_filter(id, Xor) else {
|
||||
panic!("built-in filter {id} was re-registered");
|
||||
};
|
||||
assert!(matches!(err, FormatError::FilterError(_)), "{err:?}");
|
||||
}
|
||||
assert!(builtin_filter(FILTER_SHUFFLE).is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn builtin_table_is_sorted_and_unique() {
|
||||
let ids: Vec<u16> = builtin_filters().iter().map(|f| f.id).collect();
|
||||
let mut sorted = ids.clone();
|
||||
sorted.sort_unstable();
|
||||
sorted.dedup();
|
||||
assert_eq!(ids, sorted);
|
||||
}
|
||||
}
|
||||
@@ -7,11 +7,20 @@ extern crate alloc;
|
||||
use alloc::{boxed::Box, vec, vec::Vec};
|
||||
|
||||
use crate::error::FormatError;
|
||||
#[cfg(feature = "deflate")]
|
||||
use crate::filter_pipeline::FILTER_DEFLATE;
|
||||
#[cfg(feature = "lz4")]
|
||||
use crate::filter_pipeline::FILTER_LZ4;
|
||||
#[cfg(feature = "szip")]
|
||||
use crate::filter_pipeline::FILTER_SZIP;
|
||||
#[cfg(feature = "zstd")]
|
||||
use crate::filter_pipeline::FILTER_ZSTD;
|
||||
use crate::filter_pipeline::{
|
||||
FILTER_DEFLATE, FILTER_FLETCHER32, FILTER_LZ4, FILTER_NBIT, FILTER_PCODEC,
|
||||
FILTER_PCODEC_LEGACY, FILTER_PCODEC_LEGACY_NAME, FILTER_SCALEOFFSET, FILTER_SHUFFLE,
|
||||
FILTER_SZIP, FILTER_ZSTD, FilterPipeline,
|
||||
FILTER_FLETCHER32, FILTER_NBIT, FILTER_SCALEOFFSET, FILTER_SHUFFLE, FilterPipeline,
|
||||
};
|
||||
#[cfg(feature = "pcodec")]
|
||||
use crate::filter_pipeline::{FILTER_PCODEC, FILTER_PCODEC_LEGACY, FILTER_PCODEC_LEGACY_NAME};
|
||||
use crate::filter_registry::{self, BuiltinFilter, FilterContext};
|
||||
|
||||
/// Absolute ceiling on a single decompressed chunk's output size, used only
|
||||
/// when the pipeline's declared `chunk_size` is unavailable (0). Prevents
|
||||
@@ -98,28 +107,14 @@ pub fn decompress_chunk_masked(
|
||||
if filter_skipped(filter_mask, i) {
|
||||
continue;
|
||||
}
|
||||
let bound = bounds[i];
|
||||
data = match filter.filter_id {
|
||||
FILTER_SHUFFLE => shuffle_decompress(&data, element_size as usize)?,
|
||||
// `bound` caps the decoded size so these decoders can't be forced
|
||||
// into unbounded allocation by a hostile or corrupted payload.
|
||||
FILTER_DEFLATE => deflate_decompress(&data, bound)?,
|
||||
FILTER_LZ4 => lz4_decompress(&data, bound)?,
|
||||
FILTER_ZSTD => zstd_decompress(&data, bound)?,
|
||||
FILTER_FLETCHER32 => fletcher32_verify(&data)?,
|
||||
FILTER_PCODEC => pcodec_decompress(&data, element_size as usize, bound)?,
|
||||
// Pcodec chunks written by clawhdf5 <= 2.7.0 under the ID registered
|
||||
// to Granular BitRound; recognised by the name those versions wrote.
|
||||
FILTER_PCODEC_LEGACY if filter.name.as_deref() == Some(FILTER_PCODEC_LEGACY_NAME) => {
|
||||
pcodec_decompress(&data, element_size as usize, bound)?
|
||||
}
|
||||
// These decoders also reject an element count that would
|
||||
// over-allocate past `bound`.
|
||||
FILTER_SCALEOFFSET => scaleoffset_decompress(&data, &filter.client_data, bound)?,
|
||||
FILTER_NBIT => nbit_decompress(&data, &filter.client_data, bound)?,
|
||||
FILTER_SZIP => crate::filters_szip::szip_decompress(&data, &filter.client_data, bound)?,
|
||||
other => return Err(FormatError::UnsupportedFilter(other)),
|
||||
// `max_output` caps the decoded size so a decoder can't be forced
|
||||
// into unbounded allocation by a hostile or corrupted payload.
|
||||
let ctx = FilterContext {
|
||||
filter,
|
||||
element_size: element_size as usize,
|
||||
max_output: bounds[i],
|
||||
};
|
||||
data = filter_registry::decode(&data, &ctx)?;
|
||||
}
|
||||
|
||||
Ok(data)
|
||||
@@ -135,26 +130,100 @@ pub fn compress_chunk(
|
||||
let mut result = data.to_vec();
|
||||
|
||||
for filter in &pipeline.filters {
|
||||
result = match filter.filter_id {
|
||||
FILTER_SHUFFLE => shuffle_compress(&result, element_size as usize)?,
|
||||
FILTER_DEFLATE => {
|
||||
let level = filter.client_data.first().copied().unwrap_or(6);
|
||||
deflate_compress(&result, level)?
|
||||
}
|
||||
FILTER_LZ4 => lz4_compress(&result, &filter.client_data)?,
|
||||
FILTER_ZSTD => {
|
||||
let level = filter.client_data.first().copied().unwrap_or(3);
|
||||
zstd_compress(&result, level)?
|
||||
}
|
||||
FILTER_FLETCHER32 => fletcher32_append(&result)?,
|
||||
FILTER_PCODEC => pcodec_compress(&result, element_size as usize)?,
|
||||
other => return Err(FormatError::UnsupportedFilter(other)),
|
||||
let ctx = FilterContext {
|
||||
filter,
|
||||
element_size: element_size as usize,
|
||||
max_output: 0,
|
||||
};
|
||||
result = filter_registry::encode(&result, &ctx)?;
|
||||
}
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
/// The filters compiled into this build, sorted by ID (see
|
||||
/// [`crate::filter_registry`]). A filter whose cargo feature is off is left
|
||||
/// out, so it fails as [`FormatError::UnsupportedFilter`] like any unknown ID.
|
||||
pub(crate) static BUILTIN_FILTERS: &[BuiltinFilter] = &[
|
||||
#[cfg(feature = "deflate")]
|
||||
BuiltinFilter {
|
||||
id: FILTER_DEFLATE,
|
||||
name: "deflate",
|
||||
decode: |d, c| deflate_decompress(d, c.max_output),
|
||||
encode: Some(|d, c| deflate_compress(d, c.client_data().first().copied().unwrap_or(6))),
|
||||
},
|
||||
BuiltinFilter {
|
||||
id: FILTER_SHUFFLE,
|
||||
name: "shuffle",
|
||||
decode: |d, c| shuffle_decompress(d, c.element_size),
|
||||
encode: Some(|d, c| shuffle_compress(d, c.element_size)),
|
||||
},
|
||||
BuiltinFilter {
|
||||
id: FILTER_FLETCHER32,
|
||||
name: "fletcher32",
|
||||
decode: |d, _| fletcher32_verify(d),
|
||||
encode: Some(|d, _| fletcher32_append(d)),
|
||||
},
|
||||
#[cfg(feature = "szip")]
|
||||
BuiltinFilter {
|
||||
id: FILTER_SZIP,
|
||||
name: "szip",
|
||||
decode: |d, c| crate::filters_szip::szip_decompress(d, c.client_data(), c.max_output),
|
||||
encode: None,
|
||||
},
|
||||
// These decoders also reject an element count that would over-allocate
|
||||
// past `max_output`.
|
||||
BuiltinFilter {
|
||||
id: FILTER_NBIT,
|
||||
name: "nbit",
|
||||
decode: |d, c| nbit_decompress(d, c.client_data(), c.max_output),
|
||||
encode: None,
|
||||
},
|
||||
BuiltinFilter {
|
||||
id: FILTER_SCALEOFFSET,
|
||||
name: "scaleoffset",
|
||||
decode: |d, c| scaleoffset_decompress(d, c.client_data(), c.max_output),
|
||||
encode: None,
|
||||
},
|
||||
#[cfg(feature = "pcodec")]
|
||||
BuiltinFilter {
|
||||
id: FILTER_PCODEC,
|
||||
name: "pcodec (clawhdf5 private)",
|
||||
decode: |d, c| pcodec_decompress(d, c.element_size, c.max_output),
|
||||
encode: Some(|d, c| pcodec_compress(d, c.element_size)),
|
||||
},
|
||||
#[cfg(feature = "lz4")]
|
||||
BuiltinFilter {
|
||||
id: FILTER_LZ4,
|
||||
name: "lz4",
|
||||
decode: |d, c| lz4_decompress(d, c.max_output),
|
||||
encode: Some(|d, c| lz4_compress(d, c.client_data())),
|
||||
},
|
||||
#[cfg(feature = "zstd")]
|
||||
BuiltinFilter {
|
||||
id: FILTER_ZSTD,
|
||||
name: "zstd",
|
||||
decode: |d, c| zstd_decompress(d, c.max_output),
|
||||
encode: Some(|d, c| zstd_compress(d, c.client_data().first().copied().unwrap_or(3))),
|
||||
},
|
||||
// Pcodec chunks written by clawhdf5 <= 2.7.0 under the ID registered to
|
||||
// Granular BitRound; recognised by the name those versions wrote, and
|
||||
// never written.
|
||||
#[cfg(feature = "pcodec")]
|
||||
BuiltinFilter {
|
||||
id: FILTER_PCODEC_LEGACY,
|
||||
name: "pcodec (clawhdf5 <= 2.7.0)",
|
||||
decode: |d, c| {
|
||||
if c.filter.name.as_deref() == Some(FILTER_PCODEC_LEGACY_NAME) {
|
||||
pcodec_decompress(d, c.element_size, c.max_output)
|
||||
} else {
|
||||
Err(FormatError::UnsupportedFilter(FILTER_PCODEC_LEGACY))
|
||||
}
|
||||
},
|
||||
encode: None,
|
||||
},
|
||||
];
|
||||
|
||||
/// Decode the HDF5 scale-offset filter (id 6).
|
||||
///
|
||||
/// Supports all three scale-offset variants:
|
||||
@@ -877,11 +946,6 @@ mod sysz {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "deflate"))]
|
||||
fn deflate_decompress(_data: &[u8], _expected_bytes: usize) -> Result<Vec<u8>, FormatError> {
|
||||
Err(FormatError::UnsupportedFilter(FILTER_DEFLATE))
|
||||
}
|
||||
|
||||
/// Compress data with zlib.
|
||||
#[cfg(feature = "deflate")]
|
||||
fn deflate_compress(data: &[u8], level: u32) -> Result<Vec<u8>, FormatError> {
|
||||
@@ -922,11 +986,6 @@ pub(crate) fn deflate_bounded(data: &[u8], level: u32) -> Result<Vec<u8>, String
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "deflate"))]
|
||||
fn deflate_compress(_data: &[u8], _level: u32) -> Result<Vec<u8>, FormatError> {
|
||||
Err(FormatError::UnsupportedFilter(FILTER_DEFLATE))
|
||||
}
|
||||
|
||||
/// Default LZ4 block size of the registered HDF5 LZ4 filter (`H5Zlz4.c`,
|
||||
/// `DEFAULT_BLOCK_SIZE`): 1 GiB, so an HDF5 chunk is normally one block.
|
||||
#[cfg(feature = "lz4")]
|
||||
@@ -1028,11 +1087,6 @@ fn lz4_decompress_hdf5(
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "lz4"))]
|
||||
fn lz4_decompress(_data: &[u8], _expected_bytes: usize) -> Result<Vec<u8>, FormatError> {
|
||||
Err(FormatError::UnsupportedFilter(FILTER_LZ4))
|
||||
}
|
||||
|
||||
/// Compress data in the registered HDF5 LZ4 filter format (see
|
||||
/// [`lz4_decompress`]), so libhdf5 with the LZ4 plugin (e.g. hdf5plugin) can
|
||||
/// read it. `cd[0]`, when present and non-zero, is the block size in bytes,
|
||||
@@ -1064,11 +1118,6 @@ fn lz4_compress(data: &[u8], cd: &[u32]) -> Result<Vec<u8>, FormatError> {
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "lz4"))]
|
||||
fn lz4_compress(_data: &[u8], _cd: &[u32]) -> Result<Vec<u8>, FormatError> {
|
||||
Err(FormatError::UnsupportedFilter(FILTER_LZ4))
|
||||
}
|
||||
|
||||
/// Decompress zstd data.
|
||||
///
|
||||
/// `expected_bytes` bounds the output (or [`MAX_DECOMPRESS_SIZE`] when
|
||||
@@ -1097,11 +1146,6 @@ fn zstd_decompress(data: &[u8], expected_bytes: usize) -> Result<Vec<u8>, Format
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "zstd"))]
|
||||
fn zstd_decompress(_data: &[u8], _expected_bytes: usize) -> Result<Vec<u8>, FormatError> {
|
||||
Err(FormatError::UnsupportedFilter(FILTER_ZSTD))
|
||||
}
|
||||
|
||||
/// Compress data with zstd as one frame whose header records the content
|
||||
/// size. The registered HDF5 Zstandard filter (`H5Zzstd.c`, used by
|
||||
/// libhdf5 + hdf5plugin) sizes its output buffer from
|
||||
@@ -1113,11 +1157,6 @@ fn zstd_compress(data: &[u8], level: u32) -> Result<Vec<u8>, FormatError> {
|
||||
.map_err(|e| FormatError::CompressionError(format!("zstd: {e}")))
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "zstd"))]
|
||||
fn zstd_compress(_data: &[u8], _level: u32) -> Result<Vec<u8>, FormatError> {
|
||||
Err(FormatError::UnsupportedFilter(FILTER_ZSTD))
|
||||
}
|
||||
|
||||
/// Unshuffle (decompress direction): reconstruct interleaved element bytes.
|
||||
/// On disk: all byte-0s of each element together, then all byte-1s, etc.
|
||||
/// Output: elements in natural order.
|
||||
@@ -1389,11 +1428,6 @@ fn pcodec_compress(data: &[u8], element_size: usize) -> Result<Vec<u8>, FormatEr
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "pcodec"))]
|
||||
fn pcodec_compress(_data: &[u8], _element_size: usize) -> Result<Vec<u8>, FormatError> {
|
||||
Err(FormatError::UnsupportedFilter(FILTER_PCODEC))
|
||||
}
|
||||
|
||||
/// `expected_bytes` bounds the number of elements decoded: the output buffer
|
||||
/// is pre-sized to exactly `expected_bytes / element_size` elements and
|
||||
/// `simple_decompress_into` never writes past it, so a corrupted/hostile pco
|
||||
@@ -1452,15 +1486,6 @@ fn pcodec_decompress(
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "pcodec"))]
|
||||
fn pcodec_decompress(
|
||||
_data: &[u8],
|
||||
_element_size: usize,
|
||||
_expected_bytes: usize,
|
||||
) -> Result<Vec<u8>, FormatError> {
|
||||
Err(FormatError::UnsupportedFilter(FILTER_PCODEC))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
@@ -34,6 +34,7 @@ const SZ_NN_OPTION_MASK: u32 = 32;
|
||||
///
|
||||
/// The chunk is a 4-byte little-endian uncompressed size followed by the
|
||||
/// szlib stream.
|
||||
#[cfg_attr(not(feature = "szip"), allow(dead_code))]
|
||||
pub(crate) fn szip_decompress(
|
||||
_data: &[u8],
|
||||
_cd: &[u32],
|
||||
|
||||
@@ -71,6 +71,7 @@ pub mod extensible_array;
|
||||
pub mod file_writer;
|
||||
pub mod fill_value;
|
||||
pub mod filter_pipeline;
|
||||
pub mod filter_registry;
|
||||
pub mod filters;
|
||||
mod filters_szip;
|
||||
pub mod fixed_array;
|
||||
|
||||
Reference in New Issue
Block a user