feat(format): a filter registry — filters are looked up by ID
decompress_chunk_masked and compress_chunk matched on the filter ID. They now look the ID up in filter_registry: a static table of the built-in filters compiled into this build (a filter whose cargo feature is off is simply absent), then the codecs an application registered at run time with register_filter (a FilterCodec, or a plain decoding closure). Registered codecs cannot shadow a built-in one, and their output is held to the same per-stage bound as the built-in decoders. An ID in neither tier still fails with UnsupportedFilter(id). The "feature off" stub functions that returned UnsupportedFilter are gone: the table leaves those filters out instead. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
@@ -0,0 +1,355 @@
|
|||||||
|
//! Filter registry: every filter is looked up here by its HDF5 filter ID.
|
||||||
|
//!
|
||||||
|
//! Two tiers:
|
||||||
|
//!
|
||||||
|
//! * **Built-in filters** — a static table of the filters compiled into this
|
||||||
|
//! build: the HDF5 standard filters (deflate, shuffle, Fletcher32, szip,
|
||||||
|
//! N-Bit, scale-offset) and the plugin filters whose cargo features are
|
||||||
|
//! enabled (LZ4, Zstandard, pcodec, LZF, bitshuffle, bzip2, blosc).
|
||||||
|
//! [`builtin_filters`] lists them.
|
||||||
|
//! * **Registered filters** (`std` only) — codecs the application supplies
|
||||||
|
//! for any other ID with [`register_filter`] (a [`FilterCodec`], or just a
|
||||||
|
//! decoding closure). A registered codec cannot shadow a built-in one.
|
||||||
|
//!
|
||||||
|
//! An ID in neither tier fails with [`FormatError::UnsupportedFilter`], as it
|
||||||
|
//! always has.
|
||||||
|
//!
|
||||||
|
//! ```
|
||||||
|
//! # #[cfg(feature = "std")] {
|
||||||
|
//! use clawhdf5_format::filter_registry::{self, FilterContext};
|
||||||
|
//! use clawhdf5_format::error::FormatError;
|
||||||
|
//!
|
||||||
|
//! // A toy filter in the private-use range: every byte XORed with 0x5A.
|
||||||
|
//! filter_registry::register_filter(300, |input: &[u8], _ctx: &FilterContext<'_>| {
|
||||||
|
//! Ok::<_, FormatError>(input.iter().map(|b| b ^ 0x5A).collect())
|
||||||
|
//! })
|
||||||
|
//! .unwrap();
|
||||||
|
//! assert!(filter_registry::is_filter_available(300));
|
||||||
|
//! filter_registry::unregister_filter(300);
|
||||||
|
//! # }
|
||||||
|
//! ```
|
||||||
|
|
||||||
|
#[cfg(not(feature = "std"))]
|
||||||
|
extern crate alloc;
|
||||||
|
|
||||||
|
#[cfg(not(feature = "std"))]
|
||||||
|
use alloc::vec::Vec;
|
||||||
|
|
||||||
|
use crate::error::FormatError;
|
||||||
|
use crate::filter_pipeline::FilterDescription;
|
||||||
|
|
||||||
|
/// What a codec is told about the filter it is applying.
|
||||||
|
#[derive(Debug, Clone, Copy)]
|
||||||
|
pub struct FilterContext<'a> {
|
||||||
|
/// The filter as recorded in the dataset's filter pipeline: its ID, name,
|
||||||
|
/// flags and client data (`cd_values`).
|
||||||
|
pub filter: &'a FilterDescription,
|
||||||
|
/// Size in bytes of one dataset element (the datatype's size).
|
||||||
|
pub element_size: usize,
|
||||||
|
/// Decoding only: the most bytes this stage may produce — what entered
|
||||||
|
/// the filter when the chunk was written. 0 means unknown; a decoder then
|
||||||
|
/// falls back to a fixed ceiling. Always 0 when encoding.
|
||||||
|
pub max_output: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl FilterContext<'_> {
|
||||||
|
/// The filter's client data (`cd_values`).
|
||||||
|
pub fn client_data(&self) -> &[u32] {
|
||||||
|
&self.filter.client_data
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The largest output a decoder should allow: [`Self::max_output`], or
|
||||||
|
/// 256 MiB when that is unknown.
|
||||||
|
pub fn output_limit(&self) -> usize {
|
||||||
|
if self.max_output != 0 {
|
||||||
|
self.max_output
|
||||||
|
} else {
|
||||||
|
crate::filters::MAX_DECOMPRESS_SIZE
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A filter implementation.
|
||||||
|
///
|
||||||
|
/// `decode` undoes the filter (the read direction). `encode` applies it (the
|
||||||
|
/// write direction); the default refuses with
|
||||||
|
/// [`FormatError::UnsupportedFilter`], which is right for a read-only codec.
|
||||||
|
pub trait FilterCodec: Send + Sync {
|
||||||
|
/// Undo the filter on one chunk. The output must not exceed
|
||||||
|
/// [`FilterContext::output_limit`]; the pipeline rejects a larger one.
|
||||||
|
fn decode(&self, input: &[u8], ctx: &FilterContext<'_>) -> Result<Vec<u8>, FormatError>;
|
||||||
|
|
||||||
|
/// Apply the filter to one chunk.
|
||||||
|
fn encode(&self, input: &[u8], ctx: &FilterContext<'_>) -> Result<Vec<u8>, FormatError> {
|
||||||
|
let _ = input;
|
||||||
|
Err(FormatError::UnsupportedFilter(ctx.filter.filter_id))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Any `Fn(&[u8], &FilterContext) -> Result<Vec<u8>, FormatError>` is a
|
||||||
|
/// decode-only codec.
|
||||||
|
impl<F> FilterCodec for F
|
||||||
|
where
|
||||||
|
F: Fn(&[u8], &FilterContext<'_>) -> Result<Vec<u8>, FormatError> + Send + Sync,
|
||||||
|
{
|
||||||
|
fn decode(&self, input: &[u8], ctx: &FilterContext<'_>) -> Result<Vec<u8>, FormatError> {
|
||||||
|
self(input, ctx)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Signature of a built-in filter's decoder or encoder.
|
||||||
|
pub type BuiltinFn = fn(&[u8], &FilterContext<'_>) -> Result<Vec<u8>, FormatError>;
|
||||||
|
|
||||||
|
/// A filter compiled into this build.
|
||||||
|
#[derive(Debug, Clone, Copy)]
|
||||||
|
pub struct BuiltinFilter {
|
||||||
|
/// HDF5 filter ID.
|
||||||
|
pub id: u16,
|
||||||
|
/// Human-readable name.
|
||||||
|
pub name: &'static str,
|
||||||
|
/// Decoder.
|
||||||
|
pub(crate) decode: BuiltinFn,
|
||||||
|
/// Encoder, if this build can write the filter.
|
||||||
|
pub(crate) encode: Option<BuiltinFn>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl BuiltinFilter {
|
||||||
|
/// Whether this build can write the filter as well as read it.
|
||||||
|
pub fn can_encode(&self) -> bool {
|
||||||
|
self.encode.is_some()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The filters compiled into this build, in ID order.
|
||||||
|
pub fn builtin_filters() -> &'static [BuiltinFilter] {
|
||||||
|
crate::filters::BUILTIN_FILTERS
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The built-in filter with this ID, if it is compiled in.
|
||||||
|
pub fn builtin_filter(id: u16) -> Option<&'static BuiltinFilter> {
|
||||||
|
builtin_filters().iter().find(|f| f.id == id)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Whether a chunk filtered with `id` can be decoded: a built-in filter or a
|
||||||
|
/// registered one.
|
||||||
|
pub fn is_filter_available(id: u16) -> bool {
|
||||||
|
if builtin_filter(id).is_some() {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
#[cfg(feature = "std")]
|
||||||
|
{
|
||||||
|
registered(id).is_some()
|
||||||
|
}
|
||||||
|
#[cfg(not(feature = "std"))]
|
||||||
|
{
|
||||||
|
false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "std")]
|
||||||
|
mod custom {
|
||||||
|
use super::FilterCodec;
|
||||||
|
use std::collections::BTreeMap;
|
||||||
|
use std::sync::{Arc, PoisonError, RwLock};
|
||||||
|
|
||||||
|
pub(super) type Registry = BTreeMap<u16, Arc<dyn FilterCodec>>;
|
||||||
|
|
||||||
|
static REGISTRY: RwLock<Registry> = RwLock::new(BTreeMap::new());
|
||||||
|
|
||||||
|
pub(super) fn with_read<R>(f: impl FnOnce(&Registry) -> R) -> R {
|
||||||
|
// A panic while holding the lock cannot leave the map half-updated
|
||||||
|
// (every update is a single insert/remove), so poisoning is ignored.
|
||||||
|
f(®ISTRY.read().unwrap_or_else(PoisonError::into_inner))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn with_write<R>(f: impl FnOnce(&mut Registry) -> R) -> R {
|
||||||
|
f(&mut REGISTRY.write().unwrap_or_else(PoisonError::into_inner))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Register a codec for filter `id`, process-wide. It is used for every
|
||||||
|
/// chunk read (and, if it implements [`FilterCodec::encode`], written) with
|
||||||
|
/// that filter ID, by every file.
|
||||||
|
///
|
||||||
|
/// A plain closure `Fn(&[u8], &FilterContext) -> Result<Vec<u8>, FormatError>`
|
||||||
|
/// registers a decoder. Replaces (and returns) an earlier registration for
|
||||||
|
/// the same ID. Fails with [`FormatError::FilterError`] if `id` is a built-in
|
||||||
|
/// filter of this build: those cannot be overridden.
|
||||||
|
#[cfg(feature = "std")]
|
||||||
|
pub fn register_filter<C>(
|
||||||
|
id: u16,
|
||||||
|
codec: C,
|
||||||
|
) -> Result<Option<std::sync::Arc<dyn FilterCodec>>, FormatError>
|
||||||
|
where
|
||||||
|
C: FilterCodec + 'static,
|
||||||
|
{
|
||||||
|
if let Some(builtin) = builtin_filter(id) {
|
||||||
|
return Err(FormatError::FilterError(format!(
|
||||||
|
"filter {id} ({}) is built in and cannot be re-registered",
|
||||||
|
builtin.name
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
let codec: std::sync::Arc<dyn FilterCodec> = std::sync::Arc::new(codec);
|
||||||
|
Ok(custom::with_write(|r| r.insert(id, codec)))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Remove the codec registered for `id`. Returns whether one was registered.
|
||||||
|
#[cfg(feature = "std")]
|
||||||
|
pub fn unregister_filter(id: u16) -> bool {
|
||||||
|
custom::with_write(|r| r.remove(&id).is_some())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The codec registered for `id`, if any.
|
||||||
|
#[cfg(feature = "std")]
|
||||||
|
pub fn registered(id: u16) -> Option<std::sync::Arc<dyn FilterCodec>> {
|
||||||
|
custom::with_read(|r| r.get(&id).cloned())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Undo filter `ctx.filter` on `input`: the built-in decoder if there is one,
|
||||||
|
/// else a registered one, else [`FormatError::UnsupportedFilter`].
|
||||||
|
pub(crate) fn decode(input: &[u8], ctx: &FilterContext<'_>) -> Result<Vec<u8>, FormatError> {
|
||||||
|
let id = ctx.filter.filter_id;
|
||||||
|
if let Some(builtin) = builtin_filter(id) {
|
||||||
|
return (builtin.decode)(input, ctx);
|
||||||
|
}
|
||||||
|
#[cfg(feature = "std")]
|
||||||
|
if let Some(codec) = registered(id) {
|
||||||
|
let out = codec.decode(input, ctx)?;
|
||||||
|
// A registered codec is outside our control: hold it to the same
|
||||||
|
// bound the built-in decoders enforce.
|
||||||
|
if out.len() > ctx.output_limit() {
|
||||||
|
return Err(FormatError::DecompressionError(format!(
|
||||||
|
"filter {id}: decoded {} bytes, more than the {} the chunk can hold",
|
||||||
|
out.len(),
|
||||||
|
ctx.output_limit()
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
return Ok(out);
|
||||||
|
}
|
||||||
|
Err(FormatError::UnsupportedFilter(id))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Apply filter `ctx.filter` to `input`.
|
||||||
|
pub(crate) fn encode(input: &[u8], ctx: &FilterContext<'_>) -> Result<Vec<u8>, FormatError> {
|
||||||
|
let id = ctx.filter.filter_id;
|
||||||
|
if let Some(builtin) = builtin_filter(id) {
|
||||||
|
return match builtin.encode {
|
||||||
|
Some(encode) => encode(input, ctx),
|
||||||
|
None => Err(FormatError::UnsupportedFilter(id)),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
#[cfg(feature = "std")]
|
||||||
|
if let Some(codec) = registered(id) {
|
||||||
|
return codec.encode(input, ctx);
|
||||||
|
}
|
||||||
|
Err(FormatError::UnsupportedFilter(id))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(all(test, feature = "std"))]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use crate::filter_pipeline::{FILTER_FLETCHER32, FILTER_SHUFFLE, FilterPipeline};
|
||||||
|
use crate::filters::{compress_chunk, decompress_chunk};
|
||||||
|
|
||||||
|
fn pipeline(id: u16) -> FilterPipeline {
|
||||||
|
FilterPipeline {
|
||||||
|
version: 2,
|
||||||
|
filters: vec![FilterDescription {
|
||||||
|
filter_id: id,
|
||||||
|
name: Some("test".into()),
|
||||||
|
flags: 0,
|
||||||
|
client_data: vec![7],
|
||||||
|
}],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct Xor;
|
||||||
|
impl FilterCodec for Xor {
|
||||||
|
fn decode(&self, input: &[u8], ctx: &FilterContext<'_>) -> Result<Vec<u8>, FormatError> {
|
||||||
|
let k = ctx.client_data()[0] as u8;
|
||||||
|
Ok(input.iter().map(|b| b ^ k).collect())
|
||||||
|
}
|
||||||
|
fn encode(&self, input: &[u8], ctx: &FilterContext<'_>) -> Result<Vec<u8>, FormatError> {
|
||||||
|
self.decode(input, ctx)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Each test uses its own ID: the registry is process-wide and tests run
|
||||||
|
// in parallel.
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn unknown_filter_keeps_its_error() {
|
||||||
|
let err = decompress_chunk(b"abc", &pipeline(311), 3, 1).unwrap_err();
|
||||||
|
assert_eq!(err, FormatError::UnsupportedFilter(311));
|
||||||
|
let err = compress_chunk(b"abc", &pipeline(311), 1).unwrap_err();
|
||||||
|
assert_eq!(err, FormatError::UnsupportedFilter(311));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn registered_codec_round_trips_through_the_pipeline() {
|
||||||
|
assert!(!is_filter_available(312));
|
||||||
|
assert!(register_filter(312, Xor).unwrap().is_none());
|
||||||
|
assert!(is_filter_available(312));
|
||||||
|
let data = b"hello, registry".to_vec();
|
||||||
|
let enc = compress_chunk(&data, &pipeline(312), 1).unwrap();
|
||||||
|
assert_ne!(enc, data);
|
||||||
|
assert_eq!(
|
||||||
|
decompress_chunk(&enc, &pipeline(312), data.len(), 1).unwrap(),
|
||||||
|
data
|
||||||
|
);
|
||||||
|
assert!(unregister_filter(312));
|
||||||
|
assert!(!unregister_filter(312));
|
||||||
|
assert_eq!(
|
||||||
|
decompress_chunk(&enc, &pipeline(312), data.len(), 1).unwrap_err(),
|
||||||
|
FormatError::UnsupportedFilter(312)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn closure_registers_a_decoder_only() {
|
||||||
|
register_filter(313, |input: &[u8], _ctx: &FilterContext<'_>| {
|
||||||
|
Ok(input.iter().rev().copied().collect())
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
decompress_chunk(b"abc", &pipeline(313), 3, 1).unwrap(),
|
||||||
|
b"cba"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
compress_chunk(b"abc", &pipeline(313), 1).unwrap_err(),
|
||||||
|
FormatError::UnsupportedFilter(313)
|
||||||
|
);
|
||||||
|
unregister_filter(313);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn registered_decoder_output_is_bounded() {
|
||||||
|
register_filter(314, |_input: &[u8], _ctx: &FilterContext<'_>| {
|
||||||
|
Ok(vec![0u8; 1000])
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
let err = decompress_chunk(b"abc", &pipeline(314), 10, 1).unwrap_err();
|
||||||
|
assert!(matches!(err, FormatError::DecompressionError(_)), "{err:?}");
|
||||||
|
unregister_filter(314);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn builtins_cannot_be_overridden() {
|
||||||
|
for id in [FILTER_SHUFFLE, FILTER_FLETCHER32] {
|
||||||
|
let Err(err) = register_filter(id, Xor) else {
|
||||||
|
panic!("built-in filter {id} was re-registered");
|
||||||
|
};
|
||||||
|
assert!(matches!(err, FormatError::FilterError(_)), "{err:?}");
|
||||||
|
}
|
||||||
|
assert!(builtin_filter(FILTER_SHUFFLE).is_some());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn builtin_table_is_sorted_and_unique() {
|
||||||
|
let ids: Vec<u16> = builtin_filters().iter().map(|f| f.id).collect();
|
||||||
|
let mut sorted = ids.clone();
|
||||||
|
sorted.sort_unstable();
|
||||||
|
sorted.dedup();
|
||||||
|
assert_eq!(ids, sorted);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -7,11 +7,20 @@ extern crate alloc;
|
|||||||
use alloc::{boxed::Box, vec, vec::Vec};
|
use alloc::{boxed::Box, vec, vec::Vec};
|
||||||
|
|
||||||
use crate::error::FormatError;
|
use crate::error::FormatError;
|
||||||
|
#[cfg(feature = "deflate")]
|
||||||
|
use crate::filter_pipeline::FILTER_DEFLATE;
|
||||||
|
#[cfg(feature = "lz4")]
|
||||||
|
use crate::filter_pipeline::FILTER_LZ4;
|
||||||
|
#[cfg(feature = "szip")]
|
||||||
|
use crate::filter_pipeline::FILTER_SZIP;
|
||||||
|
#[cfg(feature = "zstd")]
|
||||||
|
use crate::filter_pipeline::FILTER_ZSTD;
|
||||||
use crate::filter_pipeline::{
|
use crate::filter_pipeline::{
|
||||||
FILTER_DEFLATE, FILTER_FLETCHER32, FILTER_LZ4, FILTER_NBIT, FILTER_PCODEC,
|
FILTER_FLETCHER32, FILTER_NBIT, FILTER_SCALEOFFSET, FILTER_SHUFFLE, FilterPipeline,
|
||||||
FILTER_PCODEC_LEGACY, FILTER_PCODEC_LEGACY_NAME, FILTER_SCALEOFFSET, FILTER_SHUFFLE,
|
|
||||||
FILTER_SZIP, FILTER_ZSTD, FilterPipeline,
|
|
||||||
};
|
};
|
||||||
|
#[cfg(feature = "pcodec")]
|
||||||
|
use crate::filter_pipeline::{FILTER_PCODEC, FILTER_PCODEC_LEGACY, FILTER_PCODEC_LEGACY_NAME};
|
||||||
|
use crate::filter_registry::{self, BuiltinFilter, FilterContext};
|
||||||
|
|
||||||
/// Absolute ceiling on a single decompressed chunk's output size, used only
|
/// Absolute ceiling on a single decompressed chunk's output size, used only
|
||||||
/// when the pipeline's declared `chunk_size` is unavailable (0). Prevents
|
/// when the pipeline's declared `chunk_size` is unavailable (0). Prevents
|
||||||
@@ -98,28 +107,14 @@ pub fn decompress_chunk_masked(
|
|||||||
if filter_skipped(filter_mask, i) {
|
if filter_skipped(filter_mask, i) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
let bound = bounds[i];
|
// `max_output` caps the decoded size so a decoder can't be forced
|
||||||
data = match filter.filter_id {
|
|
||||||
FILTER_SHUFFLE => shuffle_decompress(&data, element_size as usize)?,
|
|
||||||
// `bound` caps the decoded size so these decoders can't be forced
|
|
||||||
// into unbounded allocation by a hostile or corrupted payload.
|
// into unbounded allocation by a hostile or corrupted payload.
|
||||||
FILTER_DEFLATE => deflate_decompress(&data, bound)?,
|
let ctx = FilterContext {
|
||||||
FILTER_LZ4 => lz4_decompress(&data, bound)?,
|
filter,
|
||||||
FILTER_ZSTD => zstd_decompress(&data, bound)?,
|
element_size: element_size as usize,
|
||||||
FILTER_FLETCHER32 => fletcher32_verify(&data)?,
|
max_output: bounds[i],
|
||||||
FILTER_PCODEC => pcodec_decompress(&data, element_size as usize, bound)?,
|
|
||||||
// Pcodec chunks written by clawhdf5 <= 2.7.0 under the ID registered
|
|
||||||
// to Granular BitRound; recognised by the name those versions wrote.
|
|
||||||
FILTER_PCODEC_LEGACY if filter.name.as_deref() == Some(FILTER_PCODEC_LEGACY_NAME) => {
|
|
||||||
pcodec_decompress(&data, element_size as usize, bound)?
|
|
||||||
}
|
|
||||||
// These decoders also reject an element count that would
|
|
||||||
// over-allocate past `bound`.
|
|
||||||
FILTER_SCALEOFFSET => scaleoffset_decompress(&data, &filter.client_data, bound)?,
|
|
||||||
FILTER_NBIT => nbit_decompress(&data, &filter.client_data, bound)?,
|
|
||||||
FILTER_SZIP => crate::filters_szip::szip_decompress(&data, &filter.client_data, bound)?,
|
|
||||||
other => return Err(FormatError::UnsupportedFilter(other)),
|
|
||||||
};
|
};
|
||||||
|
data = filter_registry::decode(&data, &ctx)?;
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(data)
|
Ok(data)
|
||||||
@@ -135,26 +130,100 @@ pub fn compress_chunk(
|
|||||||
let mut result = data.to_vec();
|
let mut result = data.to_vec();
|
||||||
|
|
||||||
for filter in &pipeline.filters {
|
for filter in &pipeline.filters {
|
||||||
result = match filter.filter_id {
|
let ctx = FilterContext {
|
||||||
FILTER_SHUFFLE => shuffle_compress(&result, element_size as usize)?,
|
filter,
|
||||||
FILTER_DEFLATE => {
|
element_size: element_size as usize,
|
||||||
let level = filter.client_data.first().copied().unwrap_or(6);
|
max_output: 0,
|
||||||
deflate_compress(&result, level)?
|
|
||||||
}
|
|
||||||
FILTER_LZ4 => lz4_compress(&result, &filter.client_data)?,
|
|
||||||
FILTER_ZSTD => {
|
|
||||||
let level = filter.client_data.first().copied().unwrap_or(3);
|
|
||||||
zstd_compress(&result, level)?
|
|
||||||
}
|
|
||||||
FILTER_FLETCHER32 => fletcher32_append(&result)?,
|
|
||||||
FILTER_PCODEC => pcodec_compress(&result, element_size as usize)?,
|
|
||||||
other => return Err(FormatError::UnsupportedFilter(other)),
|
|
||||||
};
|
};
|
||||||
|
result = filter_registry::encode(&result, &ctx)?;
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(result)
|
Ok(result)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The filters compiled into this build, sorted by ID (see
|
||||||
|
/// [`crate::filter_registry`]). A filter whose cargo feature is off is left
|
||||||
|
/// out, so it fails as [`FormatError::UnsupportedFilter`] like any unknown ID.
|
||||||
|
pub(crate) static BUILTIN_FILTERS: &[BuiltinFilter] = &[
|
||||||
|
#[cfg(feature = "deflate")]
|
||||||
|
BuiltinFilter {
|
||||||
|
id: FILTER_DEFLATE,
|
||||||
|
name: "deflate",
|
||||||
|
decode: |d, c| deflate_decompress(d, c.max_output),
|
||||||
|
encode: Some(|d, c| deflate_compress(d, c.client_data().first().copied().unwrap_or(6))),
|
||||||
|
},
|
||||||
|
BuiltinFilter {
|
||||||
|
id: FILTER_SHUFFLE,
|
||||||
|
name: "shuffle",
|
||||||
|
decode: |d, c| shuffle_decompress(d, c.element_size),
|
||||||
|
encode: Some(|d, c| shuffle_compress(d, c.element_size)),
|
||||||
|
},
|
||||||
|
BuiltinFilter {
|
||||||
|
id: FILTER_FLETCHER32,
|
||||||
|
name: "fletcher32",
|
||||||
|
decode: |d, _| fletcher32_verify(d),
|
||||||
|
encode: Some(|d, _| fletcher32_append(d)),
|
||||||
|
},
|
||||||
|
#[cfg(feature = "szip")]
|
||||||
|
BuiltinFilter {
|
||||||
|
id: FILTER_SZIP,
|
||||||
|
name: "szip",
|
||||||
|
decode: |d, c| crate::filters_szip::szip_decompress(d, c.client_data(), c.max_output),
|
||||||
|
encode: None,
|
||||||
|
},
|
||||||
|
// These decoders also reject an element count that would over-allocate
|
||||||
|
// past `max_output`.
|
||||||
|
BuiltinFilter {
|
||||||
|
id: FILTER_NBIT,
|
||||||
|
name: "nbit",
|
||||||
|
decode: |d, c| nbit_decompress(d, c.client_data(), c.max_output),
|
||||||
|
encode: None,
|
||||||
|
},
|
||||||
|
BuiltinFilter {
|
||||||
|
id: FILTER_SCALEOFFSET,
|
||||||
|
name: "scaleoffset",
|
||||||
|
decode: |d, c| scaleoffset_decompress(d, c.client_data(), c.max_output),
|
||||||
|
encode: None,
|
||||||
|
},
|
||||||
|
#[cfg(feature = "pcodec")]
|
||||||
|
BuiltinFilter {
|
||||||
|
id: FILTER_PCODEC,
|
||||||
|
name: "pcodec (clawhdf5 private)",
|
||||||
|
decode: |d, c| pcodec_decompress(d, c.element_size, c.max_output),
|
||||||
|
encode: Some(|d, c| pcodec_compress(d, c.element_size)),
|
||||||
|
},
|
||||||
|
#[cfg(feature = "lz4")]
|
||||||
|
BuiltinFilter {
|
||||||
|
id: FILTER_LZ4,
|
||||||
|
name: "lz4",
|
||||||
|
decode: |d, c| lz4_decompress(d, c.max_output),
|
||||||
|
encode: Some(|d, c| lz4_compress(d, c.client_data())),
|
||||||
|
},
|
||||||
|
#[cfg(feature = "zstd")]
|
||||||
|
BuiltinFilter {
|
||||||
|
id: FILTER_ZSTD,
|
||||||
|
name: "zstd",
|
||||||
|
decode: |d, c| zstd_decompress(d, c.max_output),
|
||||||
|
encode: Some(|d, c| zstd_compress(d, c.client_data().first().copied().unwrap_or(3))),
|
||||||
|
},
|
||||||
|
// Pcodec chunks written by clawhdf5 <= 2.7.0 under the ID registered to
|
||||||
|
// Granular BitRound; recognised by the name those versions wrote, and
|
||||||
|
// never written.
|
||||||
|
#[cfg(feature = "pcodec")]
|
||||||
|
BuiltinFilter {
|
||||||
|
id: FILTER_PCODEC_LEGACY,
|
||||||
|
name: "pcodec (clawhdf5 <= 2.7.0)",
|
||||||
|
decode: |d, c| {
|
||||||
|
if c.filter.name.as_deref() == Some(FILTER_PCODEC_LEGACY_NAME) {
|
||||||
|
pcodec_decompress(d, c.element_size, c.max_output)
|
||||||
|
} else {
|
||||||
|
Err(FormatError::UnsupportedFilter(FILTER_PCODEC_LEGACY))
|
||||||
|
}
|
||||||
|
},
|
||||||
|
encode: None,
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
/// Decode the HDF5 scale-offset filter (id 6).
|
/// Decode the HDF5 scale-offset filter (id 6).
|
||||||
///
|
///
|
||||||
/// Supports all three scale-offset variants:
|
/// Supports all three scale-offset variants:
|
||||||
@@ -877,11 +946,6 @@ mod sysz {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(not(feature = "deflate"))]
|
|
||||||
fn deflate_decompress(_data: &[u8], _expected_bytes: usize) -> Result<Vec<u8>, FormatError> {
|
|
||||||
Err(FormatError::UnsupportedFilter(FILTER_DEFLATE))
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Compress data with zlib.
|
/// Compress data with zlib.
|
||||||
#[cfg(feature = "deflate")]
|
#[cfg(feature = "deflate")]
|
||||||
fn deflate_compress(data: &[u8], level: u32) -> Result<Vec<u8>, FormatError> {
|
fn deflate_compress(data: &[u8], level: u32) -> Result<Vec<u8>, FormatError> {
|
||||||
@@ -922,11 +986,6 @@ pub(crate) fn deflate_bounded(data: &[u8], level: u32) -> Result<Vec<u8>, String
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(not(feature = "deflate"))]
|
|
||||||
fn deflate_compress(_data: &[u8], _level: u32) -> Result<Vec<u8>, FormatError> {
|
|
||||||
Err(FormatError::UnsupportedFilter(FILTER_DEFLATE))
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Default LZ4 block size of the registered HDF5 LZ4 filter (`H5Zlz4.c`,
|
/// Default LZ4 block size of the registered HDF5 LZ4 filter (`H5Zlz4.c`,
|
||||||
/// `DEFAULT_BLOCK_SIZE`): 1 GiB, so an HDF5 chunk is normally one block.
|
/// `DEFAULT_BLOCK_SIZE`): 1 GiB, so an HDF5 chunk is normally one block.
|
||||||
#[cfg(feature = "lz4")]
|
#[cfg(feature = "lz4")]
|
||||||
@@ -1028,11 +1087,6 @@ fn lz4_decompress_hdf5(
|
|||||||
Ok(out)
|
Ok(out)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(not(feature = "lz4"))]
|
|
||||||
fn lz4_decompress(_data: &[u8], _expected_bytes: usize) -> Result<Vec<u8>, FormatError> {
|
|
||||||
Err(FormatError::UnsupportedFilter(FILTER_LZ4))
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Compress data in the registered HDF5 LZ4 filter format (see
|
/// Compress data in the registered HDF5 LZ4 filter format (see
|
||||||
/// [`lz4_decompress`]), so libhdf5 with the LZ4 plugin (e.g. hdf5plugin) can
|
/// [`lz4_decompress`]), so libhdf5 with the LZ4 plugin (e.g. hdf5plugin) can
|
||||||
/// read it. `cd[0]`, when present and non-zero, is the block size in bytes,
|
/// read it. `cd[0]`, when present and non-zero, is the block size in bytes,
|
||||||
@@ -1064,11 +1118,6 @@ fn lz4_compress(data: &[u8], cd: &[u32]) -> Result<Vec<u8>, FormatError> {
|
|||||||
Ok(result)
|
Ok(result)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(not(feature = "lz4"))]
|
|
||||||
fn lz4_compress(_data: &[u8], _cd: &[u32]) -> Result<Vec<u8>, FormatError> {
|
|
||||||
Err(FormatError::UnsupportedFilter(FILTER_LZ4))
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Decompress zstd data.
|
/// Decompress zstd data.
|
||||||
///
|
///
|
||||||
/// `expected_bytes` bounds the output (or [`MAX_DECOMPRESS_SIZE`] when
|
/// `expected_bytes` bounds the output (or [`MAX_DECOMPRESS_SIZE`] when
|
||||||
@@ -1097,11 +1146,6 @@ fn zstd_decompress(data: &[u8], expected_bytes: usize) -> Result<Vec<u8>, Format
|
|||||||
Ok(out)
|
Ok(out)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(not(feature = "zstd"))]
|
|
||||||
fn zstd_decompress(_data: &[u8], _expected_bytes: usize) -> Result<Vec<u8>, FormatError> {
|
|
||||||
Err(FormatError::UnsupportedFilter(FILTER_ZSTD))
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Compress data with zstd as one frame whose header records the content
|
/// Compress data with zstd as one frame whose header records the content
|
||||||
/// size. The registered HDF5 Zstandard filter (`H5Zzstd.c`, used by
|
/// size. The registered HDF5 Zstandard filter (`H5Zzstd.c`, used by
|
||||||
/// libhdf5 + hdf5plugin) sizes its output buffer from
|
/// libhdf5 + hdf5plugin) sizes its output buffer from
|
||||||
@@ -1113,11 +1157,6 @@ fn zstd_compress(data: &[u8], level: u32) -> Result<Vec<u8>, FormatError> {
|
|||||||
.map_err(|e| FormatError::CompressionError(format!("zstd: {e}")))
|
.map_err(|e| FormatError::CompressionError(format!("zstd: {e}")))
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(not(feature = "zstd"))]
|
|
||||||
fn zstd_compress(_data: &[u8], _level: u32) -> Result<Vec<u8>, FormatError> {
|
|
||||||
Err(FormatError::UnsupportedFilter(FILTER_ZSTD))
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Unshuffle (decompress direction): reconstruct interleaved element bytes.
|
/// Unshuffle (decompress direction): reconstruct interleaved element bytes.
|
||||||
/// On disk: all byte-0s of each element together, then all byte-1s, etc.
|
/// On disk: all byte-0s of each element together, then all byte-1s, etc.
|
||||||
/// Output: elements in natural order.
|
/// Output: elements in natural order.
|
||||||
@@ -1389,11 +1428,6 @@ fn pcodec_compress(data: &[u8], element_size: usize) -> Result<Vec<u8>, FormatEr
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(not(feature = "pcodec"))]
|
|
||||||
fn pcodec_compress(_data: &[u8], _element_size: usize) -> Result<Vec<u8>, FormatError> {
|
|
||||||
Err(FormatError::UnsupportedFilter(FILTER_PCODEC))
|
|
||||||
}
|
|
||||||
|
|
||||||
/// `expected_bytes` bounds the number of elements decoded: the output buffer
|
/// `expected_bytes` bounds the number of elements decoded: the output buffer
|
||||||
/// is pre-sized to exactly `expected_bytes / element_size` elements and
|
/// is pre-sized to exactly `expected_bytes / element_size` elements and
|
||||||
/// `simple_decompress_into` never writes past it, so a corrupted/hostile pco
|
/// `simple_decompress_into` never writes past it, so a corrupted/hostile pco
|
||||||
@@ -1452,15 +1486,6 @@ fn pcodec_decompress(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(not(feature = "pcodec"))]
|
|
||||||
fn pcodec_decompress(
|
|
||||||
_data: &[u8],
|
|
||||||
_element_size: usize,
|
|
||||||
_expected_bytes: usize,
|
|
||||||
) -> Result<Vec<u8>, FormatError> {
|
|
||||||
Err(FormatError::UnsupportedFilter(FILTER_PCODEC))
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|||||||
@@ -34,6 +34,7 @@ const SZ_NN_OPTION_MASK: u32 = 32;
|
|||||||
///
|
///
|
||||||
/// The chunk is a 4-byte little-endian uncompressed size followed by the
|
/// The chunk is a 4-byte little-endian uncompressed size followed by the
|
||||||
/// szlib stream.
|
/// szlib stream.
|
||||||
|
#[cfg_attr(not(feature = "szip"), allow(dead_code))]
|
||||||
pub(crate) fn szip_decompress(
|
pub(crate) fn szip_decompress(
|
||||||
_data: &[u8],
|
_data: &[u8],
|
||||||
_cd: &[u32],
|
_cd: &[u32],
|
||||||
|
|||||||
@@ -71,6 +71,7 @@ pub mod extensible_array;
|
|||||||
pub mod file_writer;
|
pub mod file_writer;
|
||||||
pub mod fill_value;
|
pub mod fill_value;
|
||||||
pub mod filter_pipeline;
|
pub mod filter_pipeline;
|
||||||
|
pub mod filter_registry;
|
||||||
pub mod filters;
|
pub mod filters;
|
||||||
mod filters_szip;
|
mod filters_szip;
|
||||||
pub mod fixed_array;
|
pub mod fixed_array;
|
||||||
|
|||||||
Reference in New Issue
Block a user