Merge branch 'feat/p1-plugin-filters' into feat/p1-proof
# Conflicts: # crates/clawhdf5/tests/h5py_chunked_read_tests.rs # docs/known-issues.md
This commit is contained in:
osobh
@@ -22,6 +22,13 @@ zstd = { version = "0.13", optional = true }
|
||||
blake3 = { version = "1", optional = true }
|
||||
libaec-sys = { path = "../libaec-sys", version = "0.1", optional = true }
|
||||
pco = { version = "1.0", optional = true }
|
||||
# Pure-Rust Zstandard, for the plugin filters that embed zstd (bitshuffle,
|
||||
# blosc). The `zstd` feature (filter 32015) links libzstd instead.
|
||||
ruzstd = { version = "0.9", optional = true }
|
||||
# bzip2 with its default backend, libbz2-rs-sys: a pure-Rust port of
|
||||
# libbzip2 (no C is compiled, despite the -sys name).
|
||||
bzip2 = { version = "0.6", optional = true }
|
||||
snap = { version = "1", optional = true }
|
||||
|
||||
[dev-dependencies]
|
||||
half = { workspace = true }
|
||||
@@ -37,7 +44,7 @@ harness = false
|
||||
# Deflate backend: `zlib-rs` (pure Rust) by default. `fast-deflate` selects
|
||||
# zlib-ng instead (C, built with cmake); flate2 prefers a C zlib whenever one
|
||||
# is enabled, so turning it on anywhere in the build overrides the default.
|
||||
default = ["std", "checksum", "deflate", "provenance", "zlib-rs", "system-zlib-decompress"]
|
||||
default = ["std", "checksum", "deflate", "provenance", "zlib-rs", "system-zlib-decompress", "lzf"]
|
||||
std = []
|
||||
checksum = []
|
||||
deflate = ["flate2"]
|
||||
@@ -56,6 +63,17 @@ zstd = ["dep:zstd"]
|
||||
blake3_hash = ["blake3"]
|
||||
szip = ["libaec-sys"]
|
||||
pcodec = ["dep:pco"]
|
||||
# Plugin filters, pure Rust. LZF (32000) is h5py's built-in compression; it
|
||||
# has no dependencies, so it is on by default.
|
||||
lzf = []
|
||||
# Bitshuffle (32008), with its LZ4 and Zstandard modes.
|
||||
bitshuffle = ["lz4_flex", "ruzstd"]
|
||||
# bzip2 (307).
|
||||
bzip2 = ["dep:bzip2", "std"]
|
||||
# Blosc 1 (32001) with its BloscLZ, LZ4, Snappy, Zlib and Zstandard codecs.
|
||||
blosc = ["lz4_flex", "ruzstd", "snap", "deflate", "std"]
|
||||
# Every plugin filter above.
|
||||
plugin-filters = ["lzf", "bitshuffle", "bzip2", "blosc"]
|
||||
|
||||
[[bench]]
|
||||
name = "parallel_decompress_bench"
|
||||
|
||||
@@ -15,7 +15,7 @@ use crate::datatype::Datatype;
|
||||
use crate::error::FormatError;
|
||||
use crate::extensible_array::{ExtensibleArrayHeader, read_extensible_array_chunks};
|
||||
use crate::filter_pipeline::FilterPipeline;
|
||||
use crate::filters::{all_filters_skipped, decompress_chunk_masked};
|
||||
use crate::filters::{all_filters_skipped, decompress_chunk_exact};
|
||||
use crate::fixed_array::{FixedArrayHeader, read_fixed_array_chunks};
|
||||
#[cfg(feature = "std")]
|
||||
use std::sync::Arc;
|
||||
@@ -65,12 +65,13 @@ fn decompress_all_chunks(
|
||||
let raw_chunk = &file_data[c_addr..c_addr + size];
|
||||
|
||||
let decompressed = if let Some(pl) = pipeline {
|
||||
decompress_chunk_masked(
|
||||
decompress_chunk_exact(
|
||||
raw_chunk,
|
||||
pl,
|
||||
chunk_total_bytes,
|
||||
element_size,
|
||||
chunk_info.filter_mask,
|
||||
&chunk_info.offsets,
|
||||
)?
|
||||
} else {
|
||||
raw_chunk.to_vec()
|
||||
@@ -1061,12 +1062,13 @@ pub fn read_chunked_data_cached(
|
||||
let cache_them = total_bytes <= cache.max_bytes();
|
||||
if let Some(pl) = pipeline {
|
||||
let decode = |c: &&ChunkInfo| -> Result<Vec<u8>, FormatError> {
|
||||
decompress_chunk_masked(
|
||||
decompress_chunk_exact(
|
||||
raw_bytes(c)?,
|
||||
pl,
|
||||
chunk_total_bytes,
|
||||
elem_size as u32,
|
||||
c.filter_mask,
|
||||
&c.offsets,
|
||||
)
|
||||
};
|
||||
for batch in misses.chunks(DECODE_BATCH) {
|
||||
@@ -1332,12 +1334,13 @@ pub fn read_chunked_data_sweep(
|
||||
ensure_len(file_data, c_addr, size)?;
|
||||
let raw_chunk = &file_data[c_addr..c_addr + size];
|
||||
let dec = if let Some(pl) = pipeline {
|
||||
decompress_chunk_masked(
|
||||
decompress_chunk_exact(
|
||||
raw_chunk,
|
||||
pl,
|
||||
chunk_total_bytes,
|
||||
elem_size as u32,
|
||||
chunk_info.filter_mask,
|
||||
&coord,
|
||||
)?
|
||||
} else {
|
||||
raw_chunk.to_vec()
|
||||
@@ -1447,12 +1450,13 @@ pub fn read_chunked_data_indexed(
|
||||
ensure_len(file_data, c_addr, size)?;
|
||||
let raw_chunk = &file_data[c_addr..c_addr + size];
|
||||
let decompressed = if let Some(pl) = pipeline {
|
||||
decompress_chunk_masked(
|
||||
decompress_chunk_exact(
|
||||
raw_chunk,
|
||||
pl,
|
||||
chunk_total_bytes,
|
||||
elem_size as u32,
|
||||
*filter_mask,
|
||||
coord,
|
||||
)?
|
||||
} else {
|
||||
raw_chunk.to_vec()
|
||||
|
||||
@@ -12,8 +12,9 @@ use crate::chunk_grid::ChunkGrid;
|
||||
use crate::ea_writer;
|
||||
use crate::error::FormatError;
|
||||
use crate::filter_pipeline::{
|
||||
FILTER_DEFLATE, FILTER_FLETCHER32, FILTER_LZ4, FILTER_PCODEC, FILTER_PCODEC_NAME,
|
||||
FILTER_SHUFFLE, FILTER_ZSTD, FilterDescription, FilterPipeline,
|
||||
FILTER_BITSHUFFLE, FILTER_BLOSC, FILTER_BZIP2, FILTER_DEFLATE, FILTER_FLETCHER32, FILTER_LZ4,
|
||||
FILTER_LZF, FILTER_PCODEC, FILTER_PCODEC_NAME, FILTER_SHUFFLE, FILTER_ZSTD, FilterDescription,
|
||||
FilterPipeline,
|
||||
};
|
||||
use crate::filters::compress_chunk;
|
||||
/// Round a file offset up to the next cache-line boundary.
|
||||
@@ -48,6 +49,167 @@ pub struct ChunkOptions {
|
||||
/// Pcodec lossless numerical compression. Private, unregistered filter
|
||||
/// ID [`FILTER_PCODEC`] (480): only clawhdf5 can read it.
|
||||
pub pcodec: bool,
|
||||
/// A plugin compression filter (LZF, ...). Takes priority over the
|
||||
/// codecs above. Each needs its cargo feature to be written.
|
||||
pub plugin: Option<PluginFilter>,
|
||||
}
|
||||
|
||||
/// A compression filter from the common HDF5 plugin set, written in the
|
||||
/// format the libhdf5 plugin (h5py / hdf5plugin) reads.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
#[non_exhaustive]
|
||||
pub enum PluginFilter {
|
||||
/// LZF (filter 32000), h5py's built-in `compression="lzf"`. Needs the
|
||||
/// `lzf` feature.
|
||||
Lzf,
|
||||
/// Bitshuffle (filter 32008): a bit transpose of each block of
|
||||
/// `block_size` elements (0 = bitshuffle's default, else a multiple of
|
||||
/// 8), optionally compressed. Needs the `bitshuffle` feature.
|
||||
Bitshuffle {
|
||||
/// Block size in elements; 0 for the default.
|
||||
block_size: u32,
|
||||
/// Compression after the transpose.
|
||||
compression: BitshuffleCompression,
|
||||
},
|
||||
/// bzip2 (filter 307) at block size `level` (1-9). Needs the `bzip2`
|
||||
/// feature.
|
||||
Bzip2 {
|
||||
/// Block size 1-9 (9 = hdf5plugin's default).
|
||||
level: u32,
|
||||
},
|
||||
/// Blosc 1 (filter 32001): `codec` at `level` (0-9; 0 stores), after
|
||||
/// `shuffle`. Needs the `blosc` feature.
|
||||
Blosc {
|
||||
/// The codec inside the Blosc frame.
|
||||
codec: BloscCodec,
|
||||
/// Compression level 0-9 (0 stores the data uncompressed).
|
||||
level: u32,
|
||||
/// The shuffle Blosc applies first.
|
||||
shuffle: BloscShuffle,
|
||||
},
|
||||
}
|
||||
|
||||
/// The codec inside a Blosc frame that clawhdf5 can write. (It reads
|
||||
/// BloscLZ too, but cannot write it.)
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum BloscCodec {
|
||||
/// LZ4.
|
||||
Lz4,
|
||||
/// Snappy.
|
||||
Snappy,
|
||||
/// Zlib, at the Blosc level.
|
||||
Zlib,
|
||||
/// Zstandard (clawhdf5's pure-Rust encoder has one level, about zstd 1).
|
||||
Zstd,
|
||||
}
|
||||
|
||||
/// The shuffle Blosc applies before compressing.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum BloscShuffle {
|
||||
/// None.
|
||||
None,
|
||||
/// Byte shuffle (Blosc's default).
|
||||
Byte,
|
||||
/// Bit shuffle.
|
||||
Bit,
|
||||
}
|
||||
|
||||
/// What bitshuffle compresses its blocks with.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum BitshuffleCompression {
|
||||
/// Transpose only.
|
||||
None,
|
||||
/// LZ4 (bitshuffle's `cname="lz4"`, the common choice).
|
||||
Lz4,
|
||||
/// Zstandard. clawhdf5's pure-Rust encoder has a single level (about
|
||||
/// zstd's level 1); `level` is recorded in the file for other writers.
|
||||
Zstd {
|
||||
/// Level recorded in `cd_values[5]`.
|
||||
level: u32,
|
||||
},
|
||||
}
|
||||
|
||||
impl PluginFilter {
|
||||
/// Whether the filter reorders bytes itself, so the automatic shuffle
|
||||
/// pre-filter would only get in its way.
|
||||
fn shuffles_itself(&self) -> bool {
|
||||
match self {
|
||||
PluginFilter::Lzf => false,
|
||||
PluginFilter::Bitshuffle { .. } => true,
|
||||
PluginFilter::Bzip2 { .. } => false,
|
||||
PluginFilter::Blosc { .. } => true,
|
||||
}
|
||||
}
|
||||
|
||||
/// The pipeline entry for this filter. `chunk_bytes` is one chunk's
|
||||
/// uncompressed size (0 if unknown).
|
||||
fn description(&self, element_size: u32, chunk_bytes: u32) -> FilterDescription {
|
||||
match self {
|
||||
// h5py's lzf_set_local: filter version, liblzf version, chunk
|
||||
// size in bytes. Optional, as h5py flags it: a chunk the filter
|
||||
// cannot shrink may then be stored unfiltered.
|
||||
PluginFilter::Lzf => FilterDescription {
|
||||
filter_id: FILTER_LZF,
|
||||
name: Some("lzf".into()),
|
||||
flags: 1,
|
||||
client_data: vec![4, 0x0105, chunk_bytes],
|
||||
},
|
||||
// bshuf_h5_set_local: version 0.4, element size, block size,
|
||||
// compression (0 none, 2 LZ4, 3 Zstandard), Zstandard level.
|
||||
// hdf5-blosc's blosc_set_local: filter revision 2, Blosc format
|
||||
// 2, type size, chunk size, then level, shuffle, compressor.
|
||||
PluginFilter::Blosc {
|
||||
codec,
|
||||
level,
|
||||
shuffle,
|
||||
} => FilterDescription {
|
||||
filter_id: FILTER_BLOSC,
|
||||
name: Some("blosc".into()),
|
||||
flags: 1,
|
||||
client_data: vec![
|
||||
2,
|
||||
2,
|
||||
element_size,
|
||||
chunk_bytes,
|
||||
(*level).min(9),
|
||||
match shuffle {
|
||||
BloscShuffle::None => 0,
|
||||
BloscShuffle::Byte => 1,
|
||||
BloscShuffle::Bit => 2,
|
||||
},
|
||||
match codec {
|
||||
BloscCodec::Lz4 => 1,
|
||||
BloscCodec::Snappy => 3,
|
||||
BloscCodec::Zlib => 4,
|
||||
BloscCodec::Zstd => 5,
|
||||
},
|
||||
],
|
||||
},
|
||||
PluginFilter::Bzip2 { level } => FilterDescription {
|
||||
filter_id: FILTER_BZIP2,
|
||||
name: Some("bzip2".into()),
|
||||
flags: 1,
|
||||
client_data: vec![(*level).clamp(1, 9)],
|
||||
},
|
||||
PluginFilter::Bitshuffle {
|
||||
block_size,
|
||||
compression,
|
||||
} => {
|
||||
let mut cd = vec![0, 4, element_size, *block_size];
|
||||
match compression {
|
||||
BitshuffleCompression::None => cd.push(0),
|
||||
BitshuffleCompression::Lz4 => cd.push(2),
|
||||
BitshuffleCompression::Zstd { level } => cd.extend([3, *level]),
|
||||
}
|
||||
FilterDescription {
|
||||
filter_id: FILTER_BITSHUFFLE,
|
||||
name: Some("bitshuffle; see https://github.com/kiyo-masui/bitshuffle".into()),
|
||||
flags: 1,
|
||||
client_data: cd,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Largest chunk the automatic choice produces, in bytes.
|
||||
@@ -92,14 +254,33 @@ impl ChunkOptions {
|
||||
|| self.lz4
|
||||
|| self.zstd_level.is_some()
|
||||
|| self.pcodec
|
||||
|| self.plugin.is_some()
|
||||
}
|
||||
|
||||
/// Build a FilterPipeline from the options.
|
||||
pub fn build_pipeline(&self, element_size: u32) -> Option<FilterPipeline> {
|
||||
self.build_pipeline_for_chunk(element_size, 0)
|
||||
}
|
||||
|
||||
/// Build a FilterPipeline for chunks of `chunk_bytes` uncompressed bytes
|
||||
/// (0 if unknown). Some plugin filters record the chunk size in their
|
||||
/// client data.
|
||||
pub fn build_pipeline_for_chunk(
|
||||
&self,
|
||||
element_size: u32,
|
||||
chunk_bytes: u32,
|
||||
) -> Option<FilterPipeline> {
|
||||
let mut filters = Vec::new();
|
||||
|
||||
let has_compression =
|
||||
self.deflate_level.is_some() || self.zstd_level.is_some() || self.lz4 || self.pcodec;
|
||||
let plugin_shuffles = self
|
||||
.plugin
|
||||
.as_ref()
|
||||
.is_some_and(PluginFilter::shuffles_itself);
|
||||
let has_compression = self.deflate_level.is_some()
|
||||
|| self.zstd_level.is_some()
|
||||
|| self.lz4
|
||||
|| self.pcodec
|
||||
|| (self.plugin.is_some() && !plugin_shuffles);
|
||||
|
||||
// Shuffle before compression. Applied if explicitly requested OR if compression
|
||||
// is active and the caller hasn't disabled it — matches h5py default behavior
|
||||
@@ -113,8 +294,11 @@ impl ChunkOptions {
|
||||
});
|
||||
}
|
||||
|
||||
// Compression filters (mutually exclusive, priority: pcodec > zstd > lz4 > deflate)
|
||||
if self.pcodec {
|
||||
// Compression filters (mutually exclusive, priority: plugin > pcodec >
|
||||
// zstd > lz4 > deflate)
|
||||
if let Some(plugin) = &self.plugin {
|
||||
filters.push(plugin.description(element_size, chunk_bytes));
|
||||
} else if self.pcodec {
|
||||
filters.push(FilterDescription {
|
||||
filter_id: FILTER_PCODEC,
|
||||
name: Some(FILTER_PCODEC_NAME.into()),
|
||||
@@ -634,7 +818,12 @@ pub fn precompress_chunks(
|
||||
element_size: usize,
|
||||
options: &ChunkOptions,
|
||||
) -> Result<PrecompressedChunks, FormatError> {
|
||||
let pipeline = options.build_pipeline(element_size as u32);
|
||||
let chunk_bytes = chunk_dims
|
||||
.iter()
|
||||
.try_fold(element_size as u64, |acc, &d| acc.checked_mul(d))
|
||||
.and_then(|b| u32::try_from(b).ok())
|
||||
.unwrap_or(0);
|
||||
let pipeline = options.build_pipeline_for_chunk(element_size as u32, chunk_bytes);
|
||||
let has_filters = pipeline.is_some();
|
||||
let pipeline_message = pipeline.as_ref().map(|pl| pl.serialize());
|
||||
|
||||
@@ -1528,6 +1717,35 @@ mod tests {
|
||||
assert_eq!(pl.filters[1].client_data, vec![3]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chunk_options_pipeline_lzf() {
|
||||
let options = ChunkOptions {
|
||||
plugin: Some(PluginFilter::Lzf),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(options.is_chunked());
|
||||
let pl = options.build_pipeline_for_chunk(8, 800).unwrap();
|
||||
assert_eq!(pl.filters.len(), 2);
|
||||
assert_eq!(pl.filters[0].filter_id, FILTER_SHUFFLE);
|
||||
assert_eq!(pl.filters[1].filter_id, FILTER_LZF);
|
||||
assert_eq!(pl.filters[1].client_data, vec![4, 0x0105, 800]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chunk_options_pipeline_bitshuffle_has_no_auto_shuffle() {
|
||||
let options = ChunkOptions {
|
||||
plugin: Some(PluginFilter::Bitshuffle {
|
||||
block_size: 0,
|
||||
compression: BitshuffleCompression::Zstd { level: 5 },
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
let pl = options.build_pipeline(4).unwrap();
|
||||
assert_eq!(pl.filters.len(), 1);
|
||||
assert_eq!(pl.filters[0].filter_id, FILTER_BITSHUFFLE);
|
||||
assert_eq!(pl.filters[0].client_data, vec![0, 4, 4, 0, 3, 5]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chunk_options_zstd_priority_over_deflate() {
|
||||
let options = ChunkOptions {
|
||||
|
||||
@@ -428,9 +428,17 @@ impl fmt::Display for FormatError {
|
||||
FormatError::InvalidFilterPipelineVersion(v) => {
|
||||
write!(f, "invalid filter pipeline version: {v}")
|
||||
}
|
||||
FormatError::UnsupportedFilter(id) => {
|
||||
write!(f, "unsupported filter: {id}")
|
||||
}
|
||||
FormatError::UnsupportedFilter(id) => match crate::filter_registry::known_filter(*id) {
|
||||
Some((name, Some(feature))) => write!(
|
||||
f,
|
||||
"unsupported filter: {id} ({name}; this build lacks the `{feature}` feature)"
|
||||
),
|
||||
Some((name, None)) => write!(
|
||||
f,
|
||||
"unsupported filter: {id} ({name}, not implemented by clawhdf5)"
|
||||
),
|
||||
None => write!(f, "unsupported filter: {id}"),
|
||||
},
|
||||
FormatError::FilterError(msg) => {
|
||||
write!(f, "filter error: {msg}")
|
||||
}
|
||||
|
||||
@@ -19,6 +19,18 @@ pub const FILTER_SCALEOFFSET: u16 = 6;
|
||||
pub const FILTER_LZ4: u16 = 32004;
|
||||
/// Zstandard compression.
|
||||
pub const FILTER_ZSTD: u16 = 32015;
|
||||
/// bzip2 (registered by PyTables; hdf5plugin's `BZip2`).
|
||||
pub const FILTER_BZIP2: u16 = 307;
|
||||
/// LZF — h5py's built-in `compression="lzf"`.
|
||||
pub const FILTER_LZF: u16 = 32000;
|
||||
/// Blosc 1 (hdf5-blosc; hdf5plugin's `Blosc`).
|
||||
pub const FILTER_BLOSC: u16 = 32001;
|
||||
/// Bitshuffle, optionally with LZ4 or Zstandard (hdf5plugin's `Bitshuffle`).
|
||||
pub const FILTER_BITSHUFFLE: u16 = 32008;
|
||||
/// ZFP lossy floating-point compression (hdf5plugin's `Zfp`). Not supported.
|
||||
pub const FILTER_ZFP: u16 = 32013;
|
||||
/// Blosc 2 (hdf5plugin's `Blosc2`).
|
||||
pub const FILTER_BLOSC2: u16 = 32026;
|
||||
/// Pcodec lossless numerical codec — a **private, unregistered** clawhdf5
|
||||
/// filter. Pcodec has no ID in the HDF Group's filter registry (checked
|
||||
/// 2026-09-25, `hdf5_plugins/docs/RegisteredFilterPlugins.md`), so it uses an
|
||||
|
||||
@@ -0,0 +1,477 @@
|
||||
//! Filter registry: every filter is looked up here by its HDF5 filter ID.
|
||||
//!
|
||||
//! Two tiers:
|
||||
//!
|
||||
//! * **Built-in filters** — a static table of the filters compiled into this
|
||||
//! build: the HDF5 standard filters (deflate, shuffle, Fletcher32, szip,
|
||||
//! N-Bit, scale-offset) and the plugin filters whose cargo features are
|
||||
//! enabled (LZ4, Zstandard, pcodec, LZF, bitshuffle, bzip2, blosc).
|
||||
//! [`builtin_filters`] lists them.
|
||||
//! * **Registered filters** (`std` only) — codecs the application supplies
|
||||
//! for any other ID with [`register_filter`] (a [`FilterCodec`], or just a
|
||||
//! decoding closure). A registered codec cannot shadow a built-in one,
|
||||
//! except under 32023: that ID belongs to Granular BitRound, and the
|
||||
//! built-in entry there only reads the pcodec chunks clawhdf5 <= 2.7.0
|
||||
//! wrote (filter name `"pcodec"`), so a codec registered for 32023 handles
|
||||
//! every other chunk with that ID, and writes.
|
||||
//!
|
||||
//! An ID in neither tier fails with [`FormatError::UnsupportedFilter`], as it
|
||||
//! always has.
|
||||
//!
|
||||
//! ```
|
||||
//! # #[cfg(feature = "std")] {
|
||||
//! use clawhdf5_format::filter_registry::{self, FilterContext};
|
||||
//! use clawhdf5_format::error::FormatError;
|
||||
//!
|
||||
//! // A toy filter in the private-use range: every byte XORed with 0x5A.
|
||||
//! filter_registry::register_filter(300, |input: &[u8], _ctx: &FilterContext<'_>| {
|
||||
//! Ok::<_, FormatError>(input.iter().map(|b| b ^ 0x5A).collect())
|
||||
//! })
|
||||
//! .unwrap();
|
||||
//! assert!(filter_registry::is_filter_available(300));
|
||||
//! filter_registry::unregister_filter(300);
|
||||
//! # }
|
||||
//! ```
|
||||
|
||||
#[cfg(not(feature = "std"))]
|
||||
extern crate alloc;
|
||||
|
||||
#[cfg(not(feature = "std"))]
|
||||
use alloc::vec::Vec;
|
||||
|
||||
use crate::error::FormatError;
|
||||
use crate::filter_pipeline::FilterDescription;
|
||||
|
||||
/// What a codec is told about the filter it is applying.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct FilterContext<'a> {
|
||||
/// The filter as recorded in the dataset's filter pipeline: its ID, name,
|
||||
/// flags and client data (`cd_values`).
|
||||
pub filter: &'a FilterDescription,
|
||||
/// Size in bytes of one dataset element (the datatype's size).
|
||||
pub element_size: usize,
|
||||
/// Decoding only: the most bytes this stage may produce — what entered
|
||||
/// the filter when the chunk was written. 0 means unknown; a decoder then
|
||||
/// falls back to a fixed ceiling. Always 0 when encoding.
|
||||
pub max_output: usize,
|
||||
}
|
||||
|
||||
impl FilterContext<'_> {
|
||||
/// The filter's client data (`cd_values`).
|
||||
pub fn client_data(&self) -> &[u32] {
|
||||
&self.filter.client_data
|
||||
}
|
||||
|
||||
/// The largest output a decoder should allow: [`Self::max_output`], or
|
||||
/// 256 MiB when that is unknown.
|
||||
pub fn output_limit(&self) -> usize {
|
||||
if self.max_output != 0 {
|
||||
self.max_output
|
||||
} else {
|
||||
crate::filters::MAX_DECOMPRESS_SIZE
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A filter implementation.
|
||||
///
|
||||
/// `decode` undoes the filter (the read direction). `encode` applies it (the
|
||||
/// write direction); the default refuses with
|
||||
/// [`FormatError::UnsupportedFilter`], which is right for a read-only codec.
|
||||
pub trait FilterCodec: Send + Sync {
|
||||
/// Undo the filter on one chunk. The output must not exceed
|
||||
/// [`FilterContext::output_limit`]; the pipeline rejects a larger one.
|
||||
fn decode(&self, input: &[u8], ctx: &FilterContext<'_>) -> Result<Vec<u8>, FormatError>;
|
||||
|
||||
/// Apply the filter to one chunk.
|
||||
fn encode(&self, input: &[u8], ctx: &FilterContext<'_>) -> Result<Vec<u8>, FormatError> {
|
||||
let _ = input;
|
||||
Err(FormatError::UnsupportedFilter(ctx.filter.filter_id))
|
||||
}
|
||||
}
|
||||
|
||||
/// Any `Fn(&[u8], &FilterContext) -> Result<Vec<u8>, FormatError>` is a
|
||||
/// decode-only codec.
|
||||
impl<F> FilterCodec for F
|
||||
where
|
||||
F: Fn(&[u8], &FilterContext<'_>) -> Result<Vec<u8>, FormatError> + Send + Sync,
|
||||
{
|
||||
fn decode(&self, input: &[u8], ctx: &FilterContext<'_>) -> Result<Vec<u8>, FormatError> {
|
||||
self(input, ctx)
|
||||
}
|
||||
}
|
||||
|
||||
/// Signature of a built-in filter's decoder or encoder.
|
||||
pub type BuiltinFn = fn(&[u8], &FilterContext<'_>) -> Result<Vec<u8>, FormatError>;
|
||||
|
||||
/// A filter compiled into this build.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct BuiltinFilter {
|
||||
/// HDF5 filter ID.
|
||||
pub id: u16,
|
||||
/// Human-readable name.
|
||||
pub name: &'static str,
|
||||
/// Decoder.
|
||||
pub(crate) decode: BuiltinFn,
|
||||
/// Encoder, if this build can write the filter.
|
||||
pub(crate) encode: Option<BuiltinFn>,
|
||||
}
|
||||
|
||||
impl BuiltinFilter {
|
||||
/// Whether this build can write the filter as well as read it.
|
||||
pub fn can_encode(&self) -> bool {
|
||||
self.encode.is_some()
|
||||
}
|
||||
|
||||
/// Whether the built-in entry only borrows its ID for some chunks, so a
|
||||
/// registered codec may take the rest: the legacy pcodec entry under
|
||||
/// Granular BitRound's 32023, which claims only chunks named `"pcodec"`.
|
||||
fn is_shared(&self) -> bool {
|
||||
self.id == crate::filter_pipeline::FILTER_PCODEC_LEGACY
|
||||
}
|
||||
|
||||
/// Whether this entry decodes chunks written with `filter`.
|
||||
fn claims(&self, filter: &crate::filter_pipeline::FilterDescription) -> bool {
|
||||
!self.is_shared()
|
||||
|| filter.name.as_deref() == Some(crate::filter_pipeline::FILTER_PCODEC_LEGACY_NAME)
|
||||
}
|
||||
}
|
||||
|
||||
/// The filters compiled into this build, in ID order.
|
||||
pub fn builtin_filters() -> &'static [BuiltinFilter] {
|
||||
crate::filters::BUILTIN_FILTERS
|
||||
}
|
||||
|
||||
/// The built-in filter with this ID, if it is compiled in.
|
||||
pub fn builtin_filter(id: u16) -> Option<&'static BuiltinFilter> {
|
||||
builtin_filters().iter().find(|f| f.id == id)
|
||||
}
|
||||
|
||||
/// Why a filter ID may be missing from this build: the filter's name, and
|
||||
/// the cargo feature that provides it (`None`: clawhdf5 does not implement
|
||||
/// it — register a codec for it with [`register_filter`]). `None` for an ID
|
||||
/// clawhdf5 knows nothing about.
|
||||
pub fn known_filter(id: u16) -> Option<(&'static str, Option<&'static str>)> {
|
||||
Some(match id {
|
||||
1 => ("deflate", Some("deflate")),
|
||||
4 => ("SZIP", Some("szip")),
|
||||
307 => ("bzip2", Some("bzip2")),
|
||||
480 => ("pcodec", Some("pcodec")),
|
||||
32000 => ("LZF", Some("lzf")),
|
||||
32001 => ("Blosc", Some("blosc")),
|
||||
32004 => ("LZ4", Some("lz4")),
|
||||
32008 => ("bitshuffle", Some("bitshuffle")),
|
||||
32013 => ("ZFP", None),
|
||||
32015 => ("Zstandard", Some("zstd")),
|
||||
32019 => ("JPEG", None),
|
||||
32022 => ("BitGroom", None),
|
||||
32023 => ("Granular BitRound", None),
|
||||
32026 => ("Blosc2", None),
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
|
||||
/// Whether a chunk filtered with `id` can be decoded: a built-in filter or a
|
||||
/// registered one.
|
||||
pub fn is_filter_available(id: u16) -> bool {
|
||||
if builtin_filter(id).is_some() {
|
||||
return true;
|
||||
}
|
||||
#[cfg(feature = "std")]
|
||||
{
|
||||
registered(id).is_some()
|
||||
}
|
||||
#[cfg(not(feature = "std"))]
|
||||
{
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "std")]
|
||||
mod custom {
|
||||
use super::FilterCodec;
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::{Arc, PoisonError, RwLock};
|
||||
|
||||
pub(super) type Registry = BTreeMap<u16, Arc<dyn FilterCodec>>;
|
||||
|
||||
static REGISTRY: RwLock<Registry> = RwLock::new(BTreeMap::new());
|
||||
|
||||
pub(super) fn with_read<R>(f: impl FnOnce(&Registry) -> R) -> R {
|
||||
// A panic while holding the lock cannot leave the map half-updated
|
||||
// (every update is a single insert/remove), so poisoning is ignored.
|
||||
f(®ISTRY.read().unwrap_or_else(PoisonError::into_inner))
|
||||
}
|
||||
|
||||
pub(super) fn with_write<R>(f: impl FnOnce(&mut Registry) -> R) -> R {
|
||||
f(&mut REGISTRY.write().unwrap_or_else(PoisonError::into_inner))
|
||||
}
|
||||
}
|
||||
|
||||
/// Register a codec for filter `id`, process-wide. It is used for every
|
||||
/// chunk read (and, if it implements [`FilterCodec::encode`], written) with
|
||||
/// that filter ID, by every file.
|
||||
///
|
||||
/// A plain closure `Fn(&[u8], &FilterContext) -> Result<Vec<u8>, FormatError>`
|
||||
/// registers a decoder. Replaces (and returns) an earlier registration for
|
||||
/// the same ID. Fails with [`FormatError::FilterError`] if `id` is a built-in
|
||||
/// filter of this build: those cannot be overridden. The exception is 32023
|
||||
/// (Granular BitRound): with the `pcodec` feature the built-in entry there
|
||||
/// reads only chunks whose filter is named `"pcodec"` (clawhdf5 <= 2.7.0's
|
||||
/// files); a codec registered for 32023 decodes every other chunk with that
|
||||
/// ID and does all the writing.
|
||||
#[cfg(feature = "std")]
|
||||
pub fn register_filter<C>(
|
||||
id: u16,
|
||||
codec: C,
|
||||
) -> Result<Option<std::sync::Arc<dyn FilterCodec>>, FormatError>
|
||||
where
|
||||
C: FilterCodec + 'static,
|
||||
{
|
||||
if let Some(builtin) = builtin_filter(id).filter(|b| !b.is_shared()) {
|
||||
return Err(FormatError::FilterError(format!(
|
||||
"filter {id} ({}) is built in and cannot be re-registered",
|
||||
builtin.name
|
||||
)));
|
||||
}
|
||||
let codec: std::sync::Arc<dyn FilterCodec> = std::sync::Arc::new(codec);
|
||||
Ok(custom::with_write(|r| r.insert(id, codec)))
|
||||
}
|
||||
|
||||
/// Remove the codec registered for `id`. Returns whether one was registered.
|
||||
#[cfg(feature = "std")]
|
||||
pub fn unregister_filter(id: u16) -> bool {
|
||||
custom::with_write(|r| r.remove(&id).is_some())
|
||||
}
|
||||
|
||||
/// The codec registered for `id`, if any.
|
||||
#[cfg(feature = "std")]
|
||||
pub fn registered(id: u16) -> Option<std::sync::Arc<dyn FilterCodec>> {
|
||||
custom::with_read(|r| r.get(&id).cloned())
|
||||
}
|
||||
|
||||
/// Undo filter `ctx.filter` on `input`: the built-in decoder if there is one
|
||||
/// that claims the chunk, else a registered one, else the built-in decoder's
|
||||
/// own refusal or [`FormatError::UnsupportedFilter`].
|
||||
pub(crate) fn decode(input: &[u8], ctx: &FilterContext<'_>) -> Result<Vec<u8>, FormatError> {
|
||||
let id = ctx.filter.filter_id;
|
||||
let builtin = builtin_filter(id);
|
||||
if let Some(builtin) = builtin.filter(|b| b.claims(ctx.filter)) {
|
||||
return (builtin.decode)(input, ctx);
|
||||
}
|
||||
#[cfg(feature = "std")]
|
||||
if let Some(codec) = registered(id) {
|
||||
let out = codec.decode(input, ctx)?;
|
||||
// A registered codec is outside our control: hold it to the same
|
||||
// bound the built-in decoders enforce.
|
||||
if out.len() > ctx.output_limit() {
|
||||
return Err(FormatError::DecompressionError(format!(
|
||||
"filter {id}: decoded {} bytes, more than the {} the chunk can hold",
|
||||
out.len(),
|
||||
ctx.output_limit()
|
||||
)));
|
||||
}
|
||||
return Ok(out);
|
||||
}
|
||||
match builtin {
|
||||
Some(builtin) => (builtin.decode)(input, ctx),
|
||||
None => Err(FormatError::UnsupportedFilter(id)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Apply filter `ctx.filter` to `input`.
|
||||
pub(crate) fn encode(input: &[u8], ctx: &FilterContext<'_>) -> Result<Vec<u8>, FormatError> {
|
||||
let id = ctx.filter.filter_id;
|
||||
#[cfg(feature = "std")]
|
||||
if builtin_filter(id).is_some_and(|b| b.is_shared())
|
||||
&& let Some(codec) = registered(id)
|
||||
{
|
||||
return codec.encode(input, ctx);
|
||||
}
|
||||
if let Some(builtin) = builtin_filter(id) {
|
||||
return match builtin.encode {
|
||||
Some(encode) => encode(input, ctx),
|
||||
None => Err(FormatError::UnsupportedFilter(id)),
|
||||
};
|
||||
}
|
||||
#[cfg(feature = "std")]
|
||||
if let Some(codec) = registered(id) {
|
||||
return codec.encode(input, ctx);
|
||||
}
|
||||
Err(FormatError::UnsupportedFilter(id))
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "std"))]
|
||||
pub(crate) mod tests {
|
||||
use super::*;
|
||||
use crate::filter_pipeline::{FILTER_FLETCHER32, FILTER_SHUFFLE, FilterPipeline};
|
||||
use crate::filters::{compress_chunk, decompress_chunk};
|
||||
|
||||
fn pipeline(id: u16) -> FilterPipeline {
|
||||
FilterPipeline {
|
||||
version: 2,
|
||||
filters: vec![FilterDescription {
|
||||
filter_id: id,
|
||||
name: Some("test".into()),
|
||||
flags: 0,
|
||||
client_data: vec![7],
|
||||
}],
|
||||
}
|
||||
}
|
||||
|
||||
struct Xor;
|
||||
impl FilterCodec for Xor {
|
||||
fn decode(&self, input: &[u8], ctx: &FilterContext<'_>) -> Result<Vec<u8>, FormatError> {
|
||||
let k = ctx.client_data()[0] as u8;
|
||||
Ok(input.iter().map(|b| b ^ k).collect())
|
||||
}
|
||||
fn encode(&self, input: &[u8], ctx: &FilterContext<'_>) -> Result<Vec<u8>, FormatError> {
|
||||
self.decode(input, ctx)
|
||||
}
|
||||
}
|
||||
|
||||
// Each test uses its own ID: the registry is process-wide and tests run
|
||||
// in parallel.
|
||||
|
||||
#[test]
|
||||
fn unknown_filter_keeps_its_error() {
|
||||
let err = decompress_chunk(b"abc", &pipeline(311), 3, 1).unwrap_err();
|
||||
assert_eq!(err, FormatError::UnsupportedFilter(311));
|
||||
let err = compress_chunk(b"abc", &pipeline(311), 1).unwrap_err();
|
||||
assert_eq!(err, FormatError::UnsupportedFilter(311));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn registered_codec_round_trips_through_the_pipeline() {
|
||||
assert!(!is_filter_available(312));
|
||||
assert!(register_filter(312, Xor).unwrap().is_none());
|
||||
assert!(is_filter_available(312));
|
||||
let data = b"hello, registry".to_vec();
|
||||
let enc = compress_chunk(&data, &pipeline(312), 1).unwrap();
|
||||
assert_ne!(enc, data);
|
||||
assert_eq!(
|
||||
decompress_chunk(&enc, &pipeline(312), data.len(), 1).unwrap(),
|
||||
data
|
||||
);
|
||||
assert!(unregister_filter(312));
|
||||
assert!(!unregister_filter(312));
|
||||
assert_eq!(
|
||||
decompress_chunk(&enc, &pipeline(312), data.len(), 1).unwrap_err(),
|
||||
FormatError::UnsupportedFilter(312)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn closure_registers_a_decoder_only() {
|
||||
register_filter(313, |input: &[u8], _ctx: &FilterContext<'_>| {
|
||||
Ok(input.iter().rev().copied().collect())
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
decompress_chunk(b"abc", &pipeline(313), 3, 1).unwrap(),
|
||||
b"cba"
|
||||
);
|
||||
assert_eq!(
|
||||
compress_chunk(b"abc", &pipeline(313), 1).unwrap_err(),
|
||||
FormatError::UnsupportedFilter(313)
|
||||
);
|
||||
unregister_filter(313);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn registered_decoder_output_is_bounded() {
|
||||
register_filter(314, |_input: &[u8], _ctx: &FilterContext<'_>| {
|
||||
Ok(vec![0u8; 1000])
|
||||
})
|
||||
.unwrap();
|
||||
let err = decompress_chunk(b"abc", &pipeline(314), 10, 1).unwrap_err();
|
||||
assert!(matches!(err, FormatError::DecompressionError(_)), "{err:?}");
|
||||
unregister_filter(314);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn builtins_cannot_be_overridden() {
|
||||
for id in [FILTER_SHUFFLE, FILTER_FLETCHER32] {
|
||||
let Err(err) = register_filter(id, Xor) else {
|
||||
panic!("built-in filter {id} was re-registered");
|
||||
};
|
||||
assert!(matches!(err, FormatError::FilterError(_)), "{err:?}");
|
||||
}
|
||||
assert!(builtin_filter(FILTER_SHUFFLE).is_some());
|
||||
}
|
||||
|
||||
/// Serialises the tests that register or read filter 32023 (the
|
||||
/// registry is process-wide).
|
||||
pub(crate) static ID_32023: std::sync::Mutex<()> = std::sync::Mutex::new(());
|
||||
|
||||
/// 32023 is Granular BitRound's ID; the `pcodec` build's built-in entry
|
||||
/// there reads only clawhdf5 <= 2.7.0's pcodec chunks (named "pcodec"),
|
||||
/// so a codec can be registered for the rest, and writes with it.
|
||||
#[test]
|
||||
fn a_codec_can_be_registered_for_granular_bitround() {
|
||||
let _guard = ID_32023
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let named = |name: Option<&str>| FilterPipeline {
|
||||
version: 2,
|
||||
filters: vec![FilterDescription {
|
||||
filter_id: 32023,
|
||||
name: name.map(Into::into),
|
||||
flags: 0,
|
||||
client_data: vec![7],
|
||||
}],
|
||||
};
|
||||
let prev = register_filter(32023, Xor).expect("32023 must be registrable");
|
||||
assert!(prev.is_none());
|
||||
let data = b"granular bitround".to_vec();
|
||||
for name in [None, Some("granular_bitround"), Some("test")] {
|
||||
let pl = named(name);
|
||||
let enc = compress_chunk(&data, &pl, 1).unwrap();
|
||||
assert_ne!(enc, data);
|
||||
assert_eq!(decompress_chunk(&enc, &pl, data.len(), 1).unwrap(), data);
|
||||
}
|
||||
// clawhdf5 <= 2.7.0's pcodec chunks still go to the built-in reader.
|
||||
#[cfg(feature = "pcodec")]
|
||||
{
|
||||
let raw: Vec<u8> = (0..64)
|
||||
.flat_map(|i| (f64::from(i) * 0.5).to_le_bytes())
|
||||
.collect();
|
||||
let comp = crate::filters::pcodec_compress(&raw, 8).unwrap();
|
||||
let mut pl = named(Some("pcodec"));
|
||||
pl.filters[0].client_data = vec![8];
|
||||
assert_eq!(decompress_chunk(&comp, &pl, raw.len(), 8).unwrap(), raw);
|
||||
}
|
||||
assert!(unregister_filter(32023));
|
||||
let pl = named(None);
|
||||
assert!(matches!(
|
||||
decompress_chunk(&data, &pl, data.len(), 1),
|
||||
Err(FormatError::UnsupportedFilter(32023))
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unsupported_filter_error_names_the_filter() {
|
||||
let msg = FormatError::UnsupportedFilter(32026).to_string();
|
||||
assert!(
|
||||
msg.contains("Blosc2") && msg.contains("not implemented"),
|
||||
"{msg}"
|
||||
);
|
||||
let msg = FormatError::UnsupportedFilter(32013).to_string();
|
||||
assert!(msg.contains("ZFP"), "{msg}");
|
||||
let msg = FormatError::UnsupportedFilter(32000).to_string();
|
||||
assert!(msg.contains("LZF") && msg.contains("`lzf`"), "{msg}");
|
||||
assert_eq!(
|
||||
FormatError::UnsupportedFilter(399).to_string(),
|
||||
"unsupported filter: 399"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn builtin_table_is_sorted_and_unique() {
|
||||
let ids: Vec<u16> = builtin_filters().iter().map(|f| f.id).collect();
|
||||
let mut sorted = ids.clone();
|
||||
sorted.sort_unstable();
|
||||
sorted.dedup();
|
||||
assert_eq!(ids, sorted);
|
||||
}
|
||||
}
|
||||
@@ -4,14 +4,23 @@
|
||||
extern crate alloc;
|
||||
|
||||
#[cfg(not(feature = "std"))]
|
||||
use alloc::{boxed::Box, vec, vec::Vec};
|
||||
use alloc::{boxed::Box, format, vec, vec::Vec};
|
||||
|
||||
use crate::error::FormatError;
|
||||
#[cfg(feature = "deflate")]
|
||||
use crate::filter_pipeline::FILTER_DEFLATE;
|
||||
#[cfg(feature = "lz4")]
|
||||
use crate::filter_pipeline::FILTER_LZ4;
|
||||
#[cfg(feature = "szip")]
|
||||
use crate::filter_pipeline::FILTER_SZIP;
|
||||
#[cfg(feature = "zstd")]
|
||||
use crate::filter_pipeline::FILTER_ZSTD;
|
||||
use crate::filter_pipeline::{
|
||||
FILTER_DEFLATE, FILTER_FLETCHER32, FILTER_LZ4, FILTER_NBIT, FILTER_PCODEC,
|
||||
FILTER_PCODEC_LEGACY, FILTER_PCODEC_LEGACY_NAME, FILTER_SCALEOFFSET, FILTER_SHUFFLE,
|
||||
FILTER_SZIP, FILTER_ZSTD, FilterPipeline,
|
||||
FILTER_FLETCHER32, FILTER_NBIT, FILTER_SCALEOFFSET, FILTER_SHUFFLE, FilterPipeline,
|
||||
};
|
||||
#[cfg(feature = "pcodec")]
|
||||
use crate::filter_pipeline::{FILTER_PCODEC, FILTER_PCODEC_LEGACY, FILTER_PCODEC_LEGACY_NAME};
|
||||
use crate::filter_registry::{self, BuiltinFilter, FilterContext};
|
||||
|
||||
/// Absolute ceiling on a single decompressed chunk's output size, used only
|
||||
/// when the pipeline's declared `chunk_size` is unavailable (0). Prevents
|
||||
@@ -98,33 +107,47 @@ pub fn decompress_chunk_masked(
|
||||
if filter_skipped(filter_mask, i) {
|
||||
continue;
|
||||
}
|
||||
let bound = bounds[i];
|
||||
data = match filter.filter_id {
|
||||
FILTER_SHUFFLE => shuffle_decompress(&data, element_size as usize)?,
|
||||
// `bound` caps the decoded size so these decoders can't be forced
|
||||
// into unbounded allocation by a hostile or corrupted payload.
|
||||
FILTER_DEFLATE => deflate_decompress(&data, bound)?,
|
||||
FILTER_LZ4 => lz4_decompress(&data, bound)?,
|
||||
FILTER_ZSTD => zstd_decompress(&data, bound)?,
|
||||
FILTER_FLETCHER32 => fletcher32_verify(&data)?,
|
||||
FILTER_PCODEC => pcodec_decompress(&data, element_size as usize, bound)?,
|
||||
// Pcodec chunks written by clawhdf5 <= 2.7.0 under the ID registered
|
||||
// to Granular BitRound; recognised by the name those versions wrote.
|
||||
FILTER_PCODEC_LEGACY if filter.name.as_deref() == Some(FILTER_PCODEC_LEGACY_NAME) => {
|
||||
pcodec_decompress(&data, element_size as usize, bound)?
|
||||
}
|
||||
// These decoders also reject an element count that would
|
||||
// over-allocate past `bound`.
|
||||
FILTER_SCALEOFFSET => scaleoffset_decompress(&data, &filter.client_data, bound)?,
|
||||
FILTER_NBIT => nbit_decompress(&data, &filter.client_data, bound)?,
|
||||
FILTER_SZIP => crate::filters_szip::szip_decompress(&data, &filter.client_data, bound)?,
|
||||
other => return Err(FormatError::UnsupportedFilter(other)),
|
||||
// `max_output` caps the decoded size so a decoder can't be forced
|
||||
// into unbounded allocation by a hostile or corrupted payload.
|
||||
let ctx = FilterContext {
|
||||
filter,
|
||||
element_size: element_size as usize,
|
||||
max_output: bounds[i],
|
||||
};
|
||||
data = filter_registry::decode(&data, &ctx)?;
|
||||
}
|
||||
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
/// Decode one stored chunk of a chunked dataset: [`decompress_chunk_masked`],
|
||||
/// then require exactly `chunk_size` bytes (when `chunk_size` is known).
|
||||
///
|
||||
/// HDF5 stores every chunk at the full chunk size — edge chunks are padded
|
||||
/// before they are filtered, and an edge chunk left unfiltered is written
|
||||
/// full-size too — so a pipeline that decodes to fewer bytes means a
|
||||
/// corrupt chunk. libhdf5 fails such a read (or returns uninitialised
|
||||
/// memory); it must never read back as zeros. `coords` (the chunk's offset
|
||||
/// in the dataset) is named in the error.
|
||||
pub fn decompress_chunk_exact(
|
||||
compressed: &[u8],
|
||||
pipeline: &FilterPipeline,
|
||||
chunk_size: usize,
|
||||
element_size: u32,
|
||||
filter_mask: u32,
|
||||
coords: &[u64],
|
||||
) -> Result<Vec<u8>, FormatError> {
|
||||
let data =
|
||||
decompress_chunk_masked(compressed, pipeline, chunk_size, element_size, filter_mask)?;
|
||||
if chunk_size != 0 && data.len() != chunk_size {
|
||||
return Err(FormatError::ChunkedReadError(format!(
|
||||
"chunk at {coords:?} decoded to {} bytes, expected {chunk_size}",
|
||||
data.len()
|
||||
)));
|
||||
}
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
/// Apply a filter pipeline to compress a chunk.
|
||||
/// Filters are applied in FORWARD order for compression.
|
||||
pub fn compress_chunk(
|
||||
@@ -135,26 +158,128 @@ pub fn compress_chunk(
|
||||
let mut result = data.to_vec();
|
||||
|
||||
for filter in &pipeline.filters {
|
||||
result = match filter.filter_id {
|
||||
FILTER_SHUFFLE => shuffle_compress(&result, element_size as usize)?,
|
||||
FILTER_DEFLATE => {
|
||||
let level = filter.client_data.first().copied().unwrap_or(6);
|
||||
deflate_compress(&result, level)?
|
||||
}
|
||||
FILTER_LZ4 => lz4_compress(&result, &filter.client_data)?,
|
||||
FILTER_ZSTD => {
|
||||
let level = filter.client_data.first().copied().unwrap_or(3);
|
||||
zstd_compress(&result, level)?
|
||||
}
|
||||
FILTER_FLETCHER32 => fletcher32_append(&result)?,
|
||||
FILTER_PCODEC => pcodec_compress(&result, element_size as usize)?,
|
||||
other => return Err(FormatError::UnsupportedFilter(other)),
|
||||
let ctx = FilterContext {
|
||||
filter,
|
||||
element_size: element_size as usize,
|
||||
max_output: 0,
|
||||
};
|
||||
result = filter_registry::encode(&result, &ctx)?;
|
||||
}
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
/// The filters compiled into this build, sorted by ID (see
|
||||
/// [`crate::filter_registry`]). A filter whose cargo feature is off is left
|
||||
/// out, so it fails as [`FormatError::UnsupportedFilter`] like any unknown ID.
|
||||
pub(crate) static BUILTIN_FILTERS: &[BuiltinFilter] = &[
|
||||
#[cfg(feature = "deflate")]
|
||||
BuiltinFilter {
|
||||
id: FILTER_DEFLATE,
|
||||
name: "deflate",
|
||||
decode: |d, c| deflate_decompress(d, c.max_output),
|
||||
encode: Some(|d, c| deflate_compress(d, c.client_data().first().copied().unwrap_or(6))),
|
||||
},
|
||||
BuiltinFilter {
|
||||
id: FILTER_SHUFFLE,
|
||||
name: "shuffle",
|
||||
decode: |d, c| shuffle_decompress(d, c.element_size),
|
||||
encode: Some(|d, c| shuffle_compress(d, c.element_size)),
|
||||
},
|
||||
BuiltinFilter {
|
||||
id: FILTER_FLETCHER32,
|
||||
name: "fletcher32",
|
||||
decode: |d, _| fletcher32_verify(d),
|
||||
encode: Some(|d, _| fletcher32_append(d)),
|
||||
},
|
||||
#[cfg(feature = "szip")]
|
||||
BuiltinFilter {
|
||||
id: FILTER_SZIP,
|
||||
name: "szip",
|
||||
decode: |d, c| crate::filters_szip::szip_decompress(d, c.client_data(), c.max_output),
|
||||
encode: None,
|
||||
},
|
||||
// These decoders also reject an element count that would over-allocate
|
||||
// past `max_output`.
|
||||
BuiltinFilter {
|
||||
id: FILTER_NBIT,
|
||||
name: "nbit",
|
||||
decode: |d, c| nbit_decompress(d, c.client_data(), c.max_output),
|
||||
encode: None,
|
||||
},
|
||||
BuiltinFilter {
|
||||
id: FILTER_SCALEOFFSET,
|
||||
name: "scaleoffset",
|
||||
decode: |d, c| scaleoffset_decompress(d, c.client_data(), c.max_output),
|
||||
encode: None,
|
||||
},
|
||||
#[cfg(feature = "bzip2")]
|
||||
BuiltinFilter {
|
||||
id: crate::filter_pipeline::FILTER_BZIP2,
|
||||
name: "bzip2",
|
||||
decode: crate::filters_bzip2::bzip2_decode,
|
||||
encode: Some(crate::filters_bzip2::bzip2_encode),
|
||||
},
|
||||
#[cfg(feature = "pcodec")]
|
||||
BuiltinFilter {
|
||||
id: FILTER_PCODEC,
|
||||
name: "pcodec (clawhdf5 private)",
|
||||
decode: |d, c| pcodec_decompress(d, c.element_size, c.max_output),
|
||||
encode: Some(|d, c| pcodec_compress(d, c.element_size)),
|
||||
},
|
||||
#[cfg(feature = "lzf")]
|
||||
BuiltinFilter {
|
||||
id: crate::filter_pipeline::FILTER_LZF,
|
||||
name: "lzf",
|
||||
decode: crate::filters_lzf::lzf_decode,
|
||||
encode: Some(crate::filters_lzf::lzf_encode),
|
||||
},
|
||||
#[cfg(feature = "blosc")]
|
||||
BuiltinFilter {
|
||||
id: crate::filter_pipeline::FILTER_BLOSC,
|
||||
name: "blosc",
|
||||
decode: crate::filters_blosc::blosc_decode,
|
||||
encode: Some(crate::filters_blosc::blosc_encode),
|
||||
},
|
||||
#[cfg(feature = "lz4")]
|
||||
BuiltinFilter {
|
||||
id: FILTER_LZ4,
|
||||
name: "lz4",
|
||||
decode: |d, c| lz4_decompress(d, c.max_output),
|
||||
encode: Some(|d, c| lz4_compress(d, c.client_data())),
|
||||
},
|
||||
#[cfg(feature = "bitshuffle")]
|
||||
BuiltinFilter {
|
||||
id: crate::filter_pipeline::FILTER_BITSHUFFLE,
|
||||
name: "bitshuffle",
|
||||
decode: crate::filters_bitshuffle::bitshuffle_decode,
|
||||
encode: Some(crate::filters_bitshuffle::bitshuffle_encode),
|
||||
},
|
||||
#[cfg(feature = "zstd")]
|
||||
BuiltinFilter {
|
||||
id: FILTER_ZSTD,
|
||||
name: "zstd",
|
||||
decode: |d, c| zstd_decompress(d, c.max_output),
|
||||
encode: Some(|d, c| zstd_compress(d, c.client_data().first().copied().unwrap_or(3))),
|
||||
},
|
||||
// Pcodec chunks written by clawhdf5 <= 2.7.0 under the ID registered to
|
||||
// Granular BitRound; recognised by the name those versions wrote, and
|
||||
// never written.
|
||||
#[cfg(feature = "pcodec")]
|
||||
BuiltinFilter {
|
||||
id: FILTER_PCODEC_LEGACY,
|
||||
name: "pcodec (clawhdf5 <= 2.7.0)",
|
||||
decode: |d, c| {
|
||||
if c.filter.name.as_deref() == Some(FILTER_PCODEC_LEGACY_NAME) {
|
||||
pcodec_decompress(d, c.element_size, c.max_output)
|
||||
} else {
|
||||
Err(FormatError::UnsupportedFilter(FILTER_PCODEC_LEGACY))
|
||||
}
|
||||
},
|
||||
encode: None,
|
||||
},
|
||||
];
|
||||
|
||||
/// Decode the HDF5 scale-offset filter (id 6).
|
||||
///
|
||||
/// Supports all three scale-offset variants:
|
||||
@@ -877,11 +1002,6 @@ mod sysz {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "deflate"))]
|
||||
fn deflate_decompress(_data: &[u8], _expected_bytes: usize) -> Result<Vec<u8>, FormatError> {
|
||||
Err(FormatError::UnsupportedFilter(FILTER_DEFLATE))
|
||||
}
|
||||
|
||||
/// Compress data with zlib.
|
||||
#[cfg(feature = "deflate")]
|
||||
fn deflate_compress(data: &[u8], level: u32) -> Result<Vec<u8>, FormatError> {
|
||||
@@ -922,11 +1042,6 @@ pub(crate) fn deflate_bounded(data: &[u8], level: u32) -> Result<Vec<u8>, String
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "deflate"))]
|
||||
fn deflate_compress(_data: &[u8], _level: u32) -> Result<Vec<u8>, FormatError> {
|
||||
Err(FormatError::UnsupportedFilter(FILTER_DEFLATE))
|
||||
}
|
||||
|
||||
/// Default LZ4 block size of the registered HDF5 LZ4 filter (`H5Zlz4.c`,
|
||||
/// `DEFAULT_BLOCK_SIZE`): 1 GiB, so an HDF5 chunk is normally one block.
|
||||
#[cfg(feature = "lz4")]
|
||||
@@ -1028,11 +1143,6 @@ fn lz4_decompress_hdf5(
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "lz4"))]
|
||||
fn lz4_decompress(_data: &[u8], _expected_bytes: usize) -> Result<Vec<u8>, FormatError> {
|
||||
Err(FormatError::UnsupportedFilter(FILTER_LZ4))
|
||||
}
|
||||
|
||||
/// Compress data in the registered HDF5 LZ4 filter format (see
|
||||
/// [`lz4_decompress`]), so libhdf5 with the LZ4 plugin (e.g. hdf5plugin) can
|
||||
/// read it. `cd[0]`, when present and non-zero, is the block size in bytes,
|
||||
@@ -1064,11 +1174,6 @@ fn lz4_compress(data: &[u8], cd: &[u32]) -> Result<Vec<u8>, FormatError> {
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "lz4"))]
|
||||
fn lz4_compress(_data: &[u8], _cd: &[u32]) -> Result<Vec<u8>, FormatError> {
|
||||
Err(FormatError::UnsupportedFilter(FILTER_LZ4))
|
||||
}
|
||||
|
||||
/// Decompress zstd data.
|
||||
///
|
||||
/// `expected_bytes` bounds the output (or [`MAX_DECOMPRESS_SIZE`] when
|
||||
@@ -1097,11 +1202,6 @@ fn zstd_decompress(data: &[u8], expected_bytes: usize) -> Result<Vec<u8>, Format
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "zstd"))]
|
||||
fn zstd_decompress(_data: &[u8], _expected_bytes: usize) -> Result<Vec<u8>, FormatError> {
|
||||
Err(FormatError::UnsupportedFilter(FILTER_ZSTD))
|
||||
}
|
||||
|
||||
/// Compress data with zstd as one frame whose header records the content
|
||||
/// size. The registered HDF5 Zstandard filter (`H5Zzstd.c`, used by
|
||||
/// libhdf5 + hdf5plugin) sizes its output buffer from
|
||||
@@ -1113,11 +1213,6 @@ fn zstd_compress(data: &[u8], level: u32) -> Result<Vec<u8>, FormatError> {
|
||||
.map_err(|e| FormatError::CompressionError(format!("zstd: {e}")))
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "zstd"))]
|
||||
fn zstd_compress(_data: &[u8], _level: u32) -> Result<Vec<u8>, FormatError> {
|
||||
Err(FormatError::UnsupportedFilter(FILTER_ZSTD))
|
||||
}
|
||||
|
||||
/// Unshuffle (decompress direction): reconstruct interleaved element bytes.
|
||||
/// On disk: all byte-0s of each element together, then all byte-1s, etc.
|
||||
/// Output: elements in natural order.
|
||||
@@ -1351,7 +1446,7 @@ fn fletcher32_append(data: &[u8]) -> Result<Vec<u8>, FormatError> {
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[cfg(feature = "pcodec")]
|
||||
fn pcodec_compress(data: &[u8], element_size: usize) -> Result<Vec<u8>, FormatError> {
|
||||
pub(crate) fn pcodec_compress(data: &[u8], element_size: usize) -> Result<Vec<u8>, FormatError> {
|
||||
use pco::ChunkConfig;
|
||||
use pco::standalone::simple_compress;
|
||||
let config = ChunkConfig::default();
|
||||
@@ -1389,11 +1484,6 @@ fn pcodec_compress(data: &[u8], element_size: usize) -> Result<Vec<u8>, FormatEr
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "pcodec"))]
|
||||
fn pcodec_compress(_data: &[u8], _element_size: usize) -> Result<Vec<u8>, FormatError> {
|
||||
Err(FormatError::UnsupportedFilter(FILTER_PCODEC))
|
||||
}
|
||||
|
||||
/// `expected_bytes` bounds the number of elements decoded: the output buffer
|
||||
/// is pre-sized to exactly `expected_bytes / element_size` elements and
|
||||
/// `simple_decompress_into` never writes past it, so a corrupted/hostile pco
|
||||
@@ -1452,17 +1542,41 @@ fn pcodec_decompress(
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "pcodec"))]
|
||||
fn pcodec_decompress(
|
||||
_data: &[u8],
|
||||
_element_size: usize,
|
||||
_expected_bytes: usize,
|
||||
) -> Result<Vec<u8>, FormatError> {
|
||||
Err(FormatError::UnsupportedFilter(FILTER_PCODEC))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
|
||||
/// A chunk whose pipeline decodes to fewer bytes than the chunk holds is
|
||||
/// an error naming the chunk, never a short buffer the reader pads.
|
||||
#[test]
|
||||
fn short_decoded_chunk_is_an_error() {
|
||||
let pipeline = FilterPipeline {
|
||||
version: 2,
|
||||
filters: vec![FilterDescription {
|
||||
filter_id: FILTER_SHUFFLE,
|
||||
name: None,
|
||||
flags: 0,
|
||||
client_data: vec![4],
|
||||
}],
|
||||
};
|
||||
let full = [7u8; 32];
|
||||
assert_eq!(
|
||||
decompress_chunk_exact(&full, &pipeline, 32, 4, 0, &[8]).unwrap(),
|
||||
full
|
||||
);
|
||||
let err = decompress_chunk_exact(&full[..16], &pipeline, 32, 4, 0, &[8, 0]).unwrap_err();
|
||||
let msg = err.to_string();
|
||||
assert!(
|
||||
msg.contains("[8, 0]") && msg.contains("16") && msg.contains("32"),
|
||||
"{msg}"
|
||||
);
|
||||
// Every filter skipped: the stored bytes are the chunk, still checked.
|
||||
assert!(decompress_chunk_exact(&full[..16], &pipeline, 32, 4, 1, &[0]).is_err());
|
||||
// Unknown chunk size: not checked.
|
||||
assert_eq!(
|
||||
decompress_chunk_exact(&full[..16], &pipeline, 0, 4, 0, &[0]).unwrap(),
|
||||
&full[..16]
|
||||
);
|
||||
}
|
||||
use super::*;
|
||||
use crate::filter_pipeline::FilterDescription;
|
||||
|
||||
@@ -1677,7 +1791,7 @@ mod tests {
|
||||
// An unsupported filter is fine when the chunk skipped it.
|
||||
let unknown = FilterPipeline {
|
||||
version: 2,
|
||||
filters: vec![filter(32000), filter(FILTER_DEFLATE)],
|
||||
filters: vec![filter(32013), filter(FILTER_DEFLATE)],
|
||||
};
|
||||
assert_eq!(
|
||||
decompress_chunk_masked(&deflated, &unknown, n, 8, 0b01).unwrap(),
|
||||
@@ -2644,6 +2758,10 @@ mod tests {
|
||||
#[cfg(feature = "pcodec")]
|
||||
fn pcodec_uses_private_id_and_reads_legacy_32023() {
|
||||
use crate::chunked_write::ChunkOptions;
|
||||
#[cfg(feature = "std")]
|
||||
let _guard = crate::filter_registry::tests::ID_32023
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let opts = ChunkOptions {
|
||||
pcodec: true,
|
||||
..Default::default()
|
||||
|
||||
@@ -0,0 +1,438 @@
|
||||
//! Bitshuffle (HDF5 filter 32008) and the bit transpose it shares with blosc.
|
||||
//!
|
||||
//! **The transform.** A block of `n` elements (`n` a multiple of 8) of
|
||||
//! `es` bytes each is viewed as an `n × 8·es` bit matrix — row *i* is
|
||||
//! element *i*, column `8·j + k` is bit *k* (LSB first) of its byte *j* — and
|
||||
//! transposed: the output is `8·es` rows of `n` bits, row `8·j + k` holding
|
||||
//! bit *k* of byte *j* of every element in order, packed LSB first. That is
|
||||
//! what `bshuf_trans_bit_elem` produces (checked against hdf5plugin's
|
||||
//! library bit for bit).
|
||||
//!
|
||||
//! **The filter** (`bshuf_h5filter.c`). `cd_values`: `[0..2]` bitshuffle
|
||||
//! version, `[2]` element size, `[3]` block size in elements (0 = default:
|
||||
//! 8192 bytes' worth, rounded down to a multiple of 8, at least 128),
|
||||
//! `[4]` compression (0 none, 2 LZ4, 3 Zstandard), `[5]` Zstandard level.
|
||||
//! The chunk is cut into blocks of `block size` elements; the tail shorter
|
||||
//! than a block is transposed as one block rounded down to a multiple of 8
|
||||
//! elements, and the last `n mod 8` elements are stored as they are.
|
||||
//! Uncompressed, that is the whole chunk. Compressed, the chunk starts with a
|
||||
//! 12-byte header — the decoded size (u64 big-endian) and the block size in
|
||||
//! bytes (u32 big-endian) — and each transposed block is stored as a u32
|
||||
//! big-endian length and an LZ4 block / Zstandard frame; the untransposed
|
||||
//! tail follows the last block.
|
||||
|
||||
#[cfg(not(feature = "std"))]
|
||||
extern crate alloc;
|
||||
#[cfg(not(feature = "std"))]
|
||||
use alloc::{format, vec, vec::Vec};
|
||||
|
||||
use crate::error::FormatError;
|
||||
#[cfg(feature = "bitshuffle")]
|
||||
use crate::filter_registry::FilterContext;
|
||||
|
||||
/// Transpose an 8×8 bit matrix packed in a u64 (byte *r* = row *r*, bit *c*
|
||||
/// of that byte = column *c*). An involution.
|
||||
#[inline]
|
||||
fn transpose8(mut x: u64) -> u64 {
|
||||
let t = (x ^ (x >> 7)) & 0x00AA_00AA_00AA_00AA;
|
||||
x = x ^ t ^ (t << 7);
|
||||
let t = (x ^ (x >> 14)) & 0x0000_CCCC_0000_CCCC;
|
||||
x = x ^ t ^ (t << 14);
|
||||
let t = (x ^ (x >> 28)) & 0x0000_0000_F0F0_F0F0;
|
||||
x ^ t ^ (t << 28)
|
||||
}
|
||||
|
||||
/// Bit-transpose one block: `input` and `out` are `n * es` bytes, `n` a
|
||||
/// multiple of 8.
|
||||
pub(crate) fn bitshuffle_block(input: &[u8], out: &mut [u8], n: usize, es: usize) {
|
||||
debug_assert!(n.is_multiple_of(8) && input.len() == n * es && out.len() == n * es);
|
||||
let row = n / 8;
|
||||
for j in 0..es {
|
||||
for g in 0..row {
|
||||
let mut x = 0u64;
|
||||
for t in 0..8 {
|
||||
x |= u64::from(input[(8 * g + t) * es + j]) << (8 * t);
|
||||
}
|
||||
let y = transpose8(x);
|
||||
for k in 0..8 {
|
||||
out[(8 * j + k) * row + g] = (y >> (8 * k)) as u8;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Undo [`bitshuffle_block`].
|
||||
pub(crate) fn bitunshuffle_block(input: &[u8], out: &mut [u8], n: usize, es: usize) {
|
||||
debug_assert!(n.is_multiple_of(8) && input.len() == n * es && out.len() == n * es);
|
||||
let row = n / 8;
|
||||
for j in 0..es {
|
||||
for g in 0..row {
|
||||
let mut y = 0u64;
|
||||
for k in 0..8 {
|
||||
y |= u64::from(input[(8 * j + k) * row + g]) << (8 * k);
|
||||
}
|
||||
let x = transpose8(y);
|
||||
for t in 0..8 {
|
||||
out[(8 * g + t) * es + j] = (x >> (8 * t)) as u8;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `bshuf_default_block_size`: 8 KiB of elements, a multiple of 8, >= 128.
|
||||
#[cfg(feature = "bitshuffle")]
|
||||
fn default_block_size(es: usize) -> usize {
|
||||
((8192 / es) / 8 * 8).max(128)
|
||||
}
|
||||
|
||||
#[cfg(feature = "bitshuffle")]
|
||||
fn err(msg: &str) -> FormatError {
|
||||
FormatError::DecompressionError(format!("bitshuffle: {msg}"))
|
||||
}
|
||||
|
||||
/// `cd_values[4]`: the compression bitshuffle applies after the transpose.
|
||||
#[cfg(feature = "bitshuffle")]
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum Codec {
|
||||
None,
|
||||
Lz4,
|
||||
Zstd,
|
||||
}
|
||||
|
||||
#[cfg(feature = "bitshuffle")]
|
||||
fn codec(cd: &[u32]) -> Result<Codec, FormatError> {
|
||||
match cd.get(4).copied().unwrap_or(0) {
|
||||
0 => Ok(Codec::None),
|
||||
2 => Ok(Codec::Lz4),
|
||||
3 => Ok(Codec::Zstd),
|
||||
other => Err(FormatError::FilterError(format!(
|
||||
"bitshuffle: unknown compression {other}"
|
||||
))),
|
||||
}
|
||||
}
|
||||
|
||||
/// The element counts of the transposed blocks for `size` elements.
|
||||
#[cfg(feature = "bitshuffle")]
|
||||
fn blocks(size: usize, block: usize) -> impl Iterator<Item = usize> {
|
||||
let full = size / block;
|
||||
let last = (size % block) / 8 * 8;
|
||||
core::iter::repeat_n(block, full).chain((last > 0).then_some(last))
|
||||
}
|
||||
|
||||
/// Decode a bitshuffle-filtered chunk.
|
||||
#[cfg(feature = "bitshuffle")]
|
||||
pub(crate) fn bitshuffle_decode(
|
||||
input: &[u8],
|
||||
ctx: &FilterContext<'_>,
|
||||
) -> Result<Vec<u8>, FormatError> {
|
||||
let cd = ctx.client_data();
|
||||
let es = match cd.get(2) {
|
||||
Some(&e) if e != 0 => e as usize,
|
||||
_ => return Err(err("missing element size")),
|
||||
};
|
||||
let codec = codec(cd)?;
|
||||
let limit = ctx.output_limit();
|
||||
if codec == Codec::None {
|
||||
if input.len() > limit {
|
||||
return Err(err("output exceeds the chunk size"));
|
||||
}
|
||||
let block = match cd.get(3) {
|
||||
Some(&b) if b != 0 => b as usize,
|
||||
_ => default_block_size(es),
|
||||
};
|
||||
if !block.is_multiple_of(8) {
|
||||
return Err(err("block size is not a multiple of 8"));
|
||||
}
|
||||
if !input.len().is_multiple_of(es) {
|
||||
return Err(err("chunk is not a whole number of elements"));
|
||||
}
|
||||
let size = input.len() / es;
|
||||
let mut out = vec![0u8; input.len()];
|
||||
let mut pos = 0;
|
||||
for n in blocks(size, block) {
|
||||
let bytes = n * es;
|
||||
bitunshuffle_block(&input[pos..pos + bytes], &mut out[pos..pos + bytes], n, es);
|
||||
pos += bytes;
|
||||
}
|
||||
out[pos..].copy_from_slice(&input[pos..]);
|
||||
return Ok(out);
|
||||
}
|
||||
|
||||
let header = input.get(..12).ok_or_else(|| err("truncated header"))?;
|
||||
let total = u64::from_be_bytes(header[..8].try_into().unwrap());
|
||||
let block_bytes = u32::from_be_bytes(header[8..12].try_into().unwrap()) as usize;
|
||||
let total = usize::try_from(total)
|
||||
.ok()
|
||||
.filter(|&t| t <= limit)
|
||||
.ok_or_else(|| err("decoded size exceeds the chunk size"))?;
|
||||
if !total.is_multiple_of(es) {
|
||||
return Err(err("chunk is not a whole number of elements"));
|
||||
}
|
||||
if block_bytes == 0 || !block_bytes.is_multiple_of(es) {
|
||||
return Err(err("bad block size"));
|
||||
}
|
||||
let block = block_bytes / es;
|
||||
if !block.is_multiple_of(8) {
|
||||
return Err(err("block size is not a multiple of 8"));
|
||||
}
|
||||
let size = total / es;
|
||||
let mut out = vec![0u8; total];
|
||||
let mut tmp = vec![0u8; block_bytes.min(total)];
|
||||
let mut ip = 12usize;
|
||||
let mut op = 0usize;
|
||||
let mut zstd = None;
|
||||
for n in blocks(size, block) {
|
||||
let bytes = n * es;
|
||||
let len = input
|
||||
.get(ip..ip + 4)
|
||||
.map(|b| u32::from_be_bytes(b.try_into().unwrap()) as usize)
|
||||
.ok_or_else(|| err("truncated block header"))?;
|
||||
ip += 4;
|
||||
let comp = input
|
||||
.get(ip..ip.saturating_add(len))
|
||||
.ok_or_else(|| err("truncated block"))?;
|
||||
ip += len;
|
||||
let dst = &mut tmp[..bytes];
|
||||
let got = match codec {
|
||||
Codec::Lz4 => lz4_flex::block::decompress_into(comp, dst)
|
||||
.map_err(|e| err(&format!("lz4: {e}")))?,
|
||||
Codec::Zstd => zstd_decode_into(
|
||||
zstd.get_or_insert_with(ruzstd::decoding::FrameDecoder::new),
|
||||
comp,
|
||||
dst,
|
||||
)?,
|
||||
Codec::None => unreachable!(),
|
||||
};
|
||||
if got != bytes {
|
||||
return Err(err("block decoded to the wrong size"));
|
||||
}
|
||||
bitunshuffle_block(dst, &mut out[op..op + bytes], n, es);
|
||||
op += bytes;
|
||||
}
|
||||
let tail = total - op;
|
||||
let rest = input
|
||||
.get(ip..ip + tail)
|
||||
.ok_or_else(|| err("truncated trailing elements"))?;
|
||||
out[op..].copy_from_slice(rest);
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Decode Zstandard frames into exactly `dst`, failing if they hold more.
|
||||
#[cfg(any(feature = "bitshuffle", feature = "blosc"))]
|
||||
pub(crate) fn zstd_decode_into(
|
||||
decoder: &mut ruzstd::decoding::FrameDecoder,
|
||||
frames: &[u8],
|
||||
dst: &mut [u8],
|
||||
) -> Result<usize, FormatError> {
|
||||
decoder
|
||||
.decode_all(frames, dst)
|
||||
.map_err(|e| FormatError::DecompressionError(format!("zstd: {e}")))
|
||||
}
|
||||
|
||||
/// Compress with ruzstd. It implements one level (roughly zstd's level 1),
|
||||
/// so the requested level only matters to other encoders.
|
||||
#[cfg(any(feature = "bitshuffle", feature = "blosc"))]
|
||||
pub(crate) fn zstd_encode(data: &[u8]) -> Vec<u8> {
|
||||
ruzstd::encoding::compress_to_vec(data, ruzstd::encoding::CompressionLevel::Fastest)
|
||||
}
|
||||
|
||||
/// Encode a chunk with the bitshuffle filter.
|
||||
#[cfg(feature = "bitshuffle")]
|
||||
pub(crate) fn bitshuffle_encode(
|
||||
input: &[u8],
|
||||
ctx: &FilterContext<'_>,
|
||||
) -> Result<Vec<u8>, FormatError> {
|
||||
let cd = ctx.client_data();
|
||||
let es = match cd.get(2) {
|
||||
Some(&e) if e != 0 => e as usize,
|
||||
_ => ctx.element_size.max(1),
|
||||
};
|
||||
let codec = codec(cd)?;
|
||||
let block = match cd.get(3) {
|
||||
Some(&b) if b != 0 => b as usize,
|
||||
_ => default_block_size(es),
|
||||
};
|
||||
let cerr = |m: &str| FormatError::CompressionError(format!("bitshuffle: {m}"));
|
||||
if !block.is_multiple_of(8) {
|
||||
return Err(cerr("block size is not a multiple of 8"));
|
||||
}
|
||||
if !input.len().is_multiple_of(es) {
|
||||
return Err(cerr("chunk is not a whole number of elements"));
|
||||
}
|
||||
let size = input.len() / es;
|
||||
let mut out = Vec::with_capacity(input.len() + 12 + input.len() / 64);
|
||||
if codec != Codec::None {
|
||||
out.extend_from_slice(&(input.len() as u64).to_be_bytes());
|
||||
let block_bytes =
|
||||
u32::try_from(block * es).map_err(|_| cerr("block size does not fit in 32 bits"))?;
|
||||
out.extend_from_slice(&block_bytes.to_be_bytes());
|
||||
}
|
||||
let mut tmp = vec![0u8; (block * es).min(input.len())];
|
||||
let mut pos = 0;
|
||||
for n in blocks(size, block) {
|
||||
let bytes = n * es;
|
||||
let dst = &mut tmp[..bytes];
|
||||
bitshuffle_block(&input[pos..pos + bytes], dst, n, es);
|
||||
match codec {
|
||||
Codec::None => out.extend_from_slice(dst),
|
||||
Codec::Lz4 | Codec::Zstd => {
|
||||
let comp = if codec == Codec::Lz4 {
|
||||
lz4_flex::block::compress(dst)
|
||||
} else {
|
||||
zstd_encode(dst)
|
||||
};
|
||||
out.extend_from_slice(&(comp.len() as u32).to_be_bytes());
|
||||
out.extend_from_slice(&comp);
|
||||
}
|
||||
}
|
||||
pos += bytes;
|
||||
}
|
||||
out.extend_from_slice(&input[pos..]);
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// The definition, one bit at a time.
|
||||
fn naive(input: &[u8], n: usize, es: usize) -> Vec<u8> {
|
||||
let mut out = vec![0u8; n * es];
|
||||
for i in 0..n {
|
||||
for j in 0..es {
|
||||
for k in 0..8 {
|
||||
if input[i * es + j] >> k & 1 == 1 {
|
||||
let p = (8 * j + k) * n + i;
|
||||
out[p / 8] |= 1 << (p % 8);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transpose_matches_the_definition_and_inverts() {
|
||||
for (n, es) in [(8, 1), (16, 2), (24, 4), (128, 8), (64, 3), (8, 16)] {
|
||||
let input: Vec<u8> = (0..n * es)
|
||||
.map(|i| (i as u32).wrapping_mul(2_654_435_761).rotate_left(7) as u8)
|
||||
.collect();
|
||||
let mut out = vec![0u8; n * es];
|
||||
bitshuffle_block(&input, &mut out, n, es);
|
||||
assert_eq!(out, naive(&input, n, es), "n={n} es={es}");
|
||||
let mut back = vec![0u8; n * es];
|
||||
bitunshuffle_block(&out, &mut back, n, es);
|
||||
assert_eq!(back, input);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "bitshuffle")]
|
||||
fn ctx_for(cd: Vec<u32>) -> crate::filter_pipeline::FilterDescription {
|
||||
crate::filter_pipeline::FilterDescription {
|
||||
filter_id: crate::filter_pipeline::FILTER_BITSHUFFLE,
|
||||
name: None,
|
||||
flags: 0,
|
||||
client_data: cd,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "bitshuffle")]
|
||||
#[test]
|
||||
fn filter_round_trips_every_mode() {
|
||||
for es in [1usize, 2, 4, 8] {
|
||||
for n in [0usize, 1, 7, 8, 100, 1000, 5003] {
|
||||
let data: Vec<u8> = (0..n * es)
|
||||
.map(|i| (i % 97) as u8 ^ (i / 300) as u8)
|
||||
.collect();
|
||||
for (comp, block) in [(0, 0), (0, 16), (2, 0), (2, 64), (3, 0), (3, 1024)] {
|
||||
let f = ctx_for(vec![0, 4, es as u32, block, comp]);
|
||||
let ctx = FilterContext {
|
||||
filter: &f,
|
||||
element_size: es,
|
||||
max_output: data.len(),
|
||||
};
|
||||
let enc = bitshuffle_encode(&data, &ctx).unwrap();
|
||||
let dec = bitshuffle_decode(&enc, &ctx).unwrap();
|
||||
assert_eq!(dec, data, "es={es} n={n} comp={comp} block={block}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "bitshuffle")]
|
||||
#[test]
|
||||
fn rejects_oversized_and_truncated_chunks() {
|
||||
let data = vec![5u8; 4096];
|
||||
let f = ctx_for(vec![0, 4, 4, 0, 2]);
|
||||
let mut ctx = FilterContext {
|
||||
filter: &f,
|
||||
element_size: 4,
|
||||
max_output: data.len(),
|
||||
};
|
||||
let enc = bitshuffle_encode(&data, &ctx).unwrap();
|
||||
assert!(bitshuffle_decode(&enc[..enc.len() - 1], &ctx).is_err());
|
||||
ctx.max_output = 100;
|
||||
assert!(bitshuffle_decode(&enc, &ctx).is_err());
|
||||
}
|
||||
|
||||
/// Random and mutated chunks, in every mode, and hostile `cd_values`:
|
||||
/// errors are fine, panics are not.
|
||||
#[cfg(feature = "bitshuffle")]
|
||||
#[test]
|
||||
fn fuzzed_chunks_never_panic() {
|
||||
use crate::test_fuzz::{Rng, fuzz_decoder};
|
||||
let data: Vec<u8> = (0..3001u32)
|
||||
.flat_map(|i| ((i / 7) as u16).to_le_bytes())
|
||||
.collect();
|
||||
for (comp, block) in [(0, 0), (0, 16), (2, 0), (2, 64), (3, 0), (3, 1024)] {
|
||||
let f = ctx_for(vec![0, 4, 2, block, comp]);
|
||||
let ctx = FilterContext {
|
||||
filter: &f,
|
||||
element_size: 2,
|
||||
max_output: data.len(),
|
||||
};
|
||||
let seeds = vec![
|
||||
bitshuffle_encode(&data, &ctx).unwrap(),
|
||||
bitshuffle_encode(&data[..34], &ctx).unwrap(),
|
||||
bitshuffle_encode(&data[..512], &ctx).unwrap(),
|
||||
];
|
||||
fuzz_decoder(
|
||||
0xb5 + comp as u64 * 7 + block as u64,
|
||||
&seeds,
|
||||
4_000,
|
||||
data.len(),
|
||||
|s| bitshuffle_decode(s, &ctx),
|
||||
);
|
||||
}
|
||||
// Hostile filter parameters on a valid chunk.
|
||||
let mut rng = Rng::new(0xcd);
|
||||
let good = ctx_for(vec![0, 4, 2, 0, 2]);
|
||||
let enc = bitshuffle_encode(
|
||||
&data,
|
||||
&FilterContext {
|
||||
filter: &good,
|
||||
element_size: 2,
|
||||
max_output: data.len(),
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
for _ in 0..3_000 {
|
||||
let cd: Vec<u32> = (0..rng.below(7))
|
||||
.map(|_| match rng.below(4) {
|
||||
0 => rng.below(5) as u32,
|
||||
1 => u32::MAX - rng.below(4) as u32,
|
||||
2 => 1 << rng.below(32),
|
||||
_ => rng.next_u64() as u32,
|
||||
})
|
||||
.collect();
|
||||
let f = ctx_for(cd);
|
||||
let ctx = FilterContext {
|
||||
filter: &f,
|
||||
element_size: 2,
|
||||
max_output: data.len(),
|
||||
};
|
||||
let _ = bitshuffle_decode(&enc, &ctx);
|
||||
let _ = bitshuffle_decode(&data, &ctx);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,711 @@
|
||||
//! Blosc 1 (HDF5 filter 32001, `hdf5-blosc`, hdf5plugin's `Blosc`), in pure
|
||||
//! Rust: the Blosc 1 frame, its byte shuffle and bit shuffle, and the
|
||||
//! BloscLZ, LZ4/LZ4HC, Snappy, Zlib and Zstandard codecs inside it.
|
||||
//!
|
||||
//! **Frame** (c-blosc 1.x, format version 2). A 16-byte header — version
|
||||
//! (2), codec format version (1), flags, type size, then little-endian `u32`
|
||||
//! decoded size, block size and frame size. Flags: bit 0 byte shuffle, bit
|
||||
//! 1 stored raw ("memcpyed": the data follows the header), bit 2 bit
|
||||
//! shuffle, bit 4 "do not split", bits 5-7 the codec (0 BloscLZ, 1 LZ4 and
|
||||
//! LZ4HC, 2 Snappy, 3 Zlib, 4 Zstandard). Unless stored raw, a table of
|
||||
//! `u32` block offsets follows, one per block of `block size` bytes (the
|
||||
//! last one may be shorter). A block is one stream, or — when the "do not
|
||||
//! split" flag is clear, the type size is at most 16, the block holds at
|
||||
//! least 128 elements, and it is not the short last block — `type size`
|
||||
//! streams, one per byte plane. Each stream is a `u32` length and the
|
||||
//! codec's output; a length equal to the stream's decoded size means the
|
||||
//! bytes are stored raw. The decoded block is then unshuffled (byte shuffle
|
||||
//! for type size > 1; bit shuffle when the block holds a multiple of 8
|
||||
//! elements, the trailing partial element copied as is).
|
||||
//!
|
||||
//! **Filter** (`blosc_filter.c`) `cd_values`: `[0]` filter revision, `[1]`
|
||||
//! Blosc format version, `[2]` type size, `[3]` chunk size in bytes, `[4]`
|
||||
//! compression level, `[5]` shuffle (0 none, 1 byte, 2 bit), `[6]`
|
||||
//! compressor (0 blosclz, 1 lz4, 2 lz4hc, 3 snappy, 4 zlib, 5 zstd). The
|
||||
//! decoder needs only the frame.
|
||||
|
||||
use crate::error::FormatError;
|
||||
use crate::filter_registry::FilterContext;
|
||||
use crate::filters_bitshuffle::{bitshuffle_block, bitunshuffle_block};
|
||||
|
||||
const HEADER: usize = 16;
|
||||
const FLAG_SHUFFLE: u8 = 0x01;
|
||||
const FLAG_MEMCPYED: u8 = 0x02;
|
||||
const FLAG_BITSHUFFLE: u8 = 0x04;
|
||||
const FLAG_FUTURE: u8 = 0x08;
|
||||
const FLAG_DONT_SPLIT: u8 = 0x10;
|
||||
const MAX_SPLITS: usize = 16;
|
||||
const MIN_BUFFERSIZE: usize = 128;
|
||||
|
||||
fn err(msg: &str) -> FormatError {
|
||||
FormatError::DecompressionError(format!("blosc: {msg}"))
|
||||
}
|
||||
|
||||
fn le32(b: &[u8], at: usize) -> Result<usize, FormatError> {
|
||||
b.get(at..at + 4)
|
||||
.map(|s| u32::from_le_bytes(s.try_into().unwrap()) as usize)
|
||||
.ok_or_else(|| err("truncated frame"))
|
||||
}
|
||||
|
||||
/// The codec inside a Blosc frame (flags bits 5-7).
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum Codec {
|
||||
BloscLz,
|
||||
Lz4,
|
||||
Snappy,
|
||||
Zlib,
|
||||
Zstd,
|
||||
}
|
||||
|
||||
impl Codec {
|
||||
fn from_flags(flags: u8) -> Result<Codec, FormatError> {
|
||||
match flags >> 5 {
|
||||
0 => Ok(Codec::BloscLz),
|
||||
1 => Ok(Codec::Lz4),
|
||||
2 => Ok(Codec::Snappy),
|
||||
3 => Ok(Codec::Zlib),
|
||||
4 => Ok(Codec::Zstd),
|
||||
other => Err(err(&format!("unknown codec {other}"))),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Decode one codec stream into exactly `dst`.
|
||||
fn decode_stream(
|
||||
codec: Codec,
|
||||
src: &[u8],
|
||||
dst: &mut [u8],
|
||||
zstd: &mut Option<ruzstd::decoding::FrameDecoder>,
|
||||
) -> Result<(), FormatError> {
|
||||
let n = match codec {
|
||||
Codec::BloscLz => blosclz_decompress(src, dst),
|
||||
Codec::Lz4 => {
|
||||
lz4_flex::block::decompress_into(src, dst).map_err(|e| err(&format!("lz4: {e}")))?
|
||||
}
|
||||
Codec::Snappy => {
|
||||
let len = snap::raw::decompress_len(src).map_err(|e| err(&format!("snappy: {e}")))?;
|
||||
if len != dst.len() {
|
||||
return Err(err("snappy stream has the wrong size"));
|
||||
}
|
||||
snap::raw::Decoder::new()
|
||||
.decompress(src, dst)
|
||||
.map_err(|e| err(&format!("snappy: {e}")))?
|
||||
}
|
||||
Codec::Zlib => {
|
||||
let out = crate::filters::inflate_bounded(src, dst.len(), dst.len())
|
||||
.map_err(|e| err(&format!("zlib: {e}")))?;
|
||||
let n = out.len();
|
||||
if n == dst.len() {
|
||||
dst.copy_from_slice(&out);
|
||||
}
|
||||
n
|
||||
}
|
||||
Codec::Zstd => crate::filters_bitshuffle::zstd_decode_into(
|
||||
zstd.get_or_insert_with(ruzstd::decoding::FrameDecoder::new),
|
||||
src,
|
||||
dst,
|
||||
)?,
|
||||
};
|
||||
if n != dst.len() {
|
||||
return Err(err("stream decoded to the wrong size"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Decode a Blosc-filtered chunk: one Blosc 1 frame.
|
||||
///
|
||||
/// An HDF5 chunk is never empty, so a frame that decodes to nothing where
|
||||
/// the chunk size is known is corrupt (libhdf5's filter fails it too).
|
||||
pub(crate) fn blosc_decode(input: &[u8], ctx: &FilterContext<'_>) -> Result<Vec<u8>, FormatError> {
|
||||
let out = blosc_decompress(input, ctx.output_limit())?;
|
||||
if out.is_empty() && ctx.max_output != 0 {
|
||||
return Err(err("empty frame for a non-empty chunk"));
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Decompress a Blosc 1 frame, refusing more than `limit` bytes of output.
|
||||
pub fn blosc_decompress(input: &[u8], limit: usize) -> Result<Vec<u8>, FormatError> {
|
||||
if input.len() < HEADER {
|
||||
return Err(err("truncated header"));
|
||||
}
|
||||
let version = input[0];
|
||||
let codec_version = input[1];
|
||||
let flags = input[2];
|
||||
let typesize = input[3] as usize;
|
||||
let nbytes = le32(input, 4)?;
|
||||
let blocksize = le32(input, 8)?;
|
||||
let cbytes = le32(input, 12)?;
|
||||
if version != 1 && version != 2 {
|
||||
return Err(err(&format!(
|
||||
"frame format version {version} is not Blosc 1 (a Blosc 2 chunk?)"
|
||||
)));
|
||||
}
|
||||
if flags & FLAG_FUTURE != 0 {
|
||||
return Err(err("unknown header flags"));
|
||||
}
|
||||
if nbytes > limit {
|
||||
return Err(err("decoded size exceeds the chunk size"));
|
||||
}
|
||||
if cbytes > input.len() {
|
||||
return Err(err("frame is longer than the chunk"));
|
||||
}
|
||||
if cbytes < HEADER {
|
||||
return Err(err("truncated frame"));
|
||||
}
|
||||
let src = &input[..cbytes];
|
||||
if nbytes == 0 {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
if blocksize == 0 || typesize == 0 {
|
||||
return Err(err("bad block or type size"));
|
||||
}
|
||||
let mut out = vec![0u8; nbytes];
|
||||
if flags & FLAG_MEMCPYED != 0 {
|
||||
if cbytes != nbytes + HEADER {
|
||||
return Err(err("stored frame has the wrong size"));
|
||||
}
|
||||
out.copy_from_slice(&src[HEADER..]);
|
||||
return Ok(out);
|
||||
}
|
||||
let codec = Codec::from_flags(flags)?;
|
||||
if codec_version != 1 {
|
||||
return Err(err(&format!(
|
||||
"unsupported {codec:?} format version {codec_version}"
|
||||
)));
|
||||
}
|
||||
let nblocks = nbytes.div_ceil(blocksize);
|
||||
let leftover = nbytes % blocksize;
|
||||
if nblocks > (cbytes - HEADER) / 4 {
|
||||
return Err(err("block table is truncated"));
|
||||
}
|
||||
let block_len = blocksize.min(nbytes);
|
||||
let mut tmp = vec![0u8; block_len];
|
||||
let mut zstd = None;
|
||||
let dont_split = flags & FLAG_DONT_SPLIT != 0;
|
||||
for j in 0..nblocks {
|
||||
let is_leftover = j == nblocks - 1 && leftover > 0;
|
||||
let bsize = if is_leftover { leftover } else { blocksize };
|
||||
let nsplits = if !dont_split
|
||||
&& typesize <= MAX_SPLITS
|
||||
&& bsize / typesize >= MIN_BUFFERSIZE
|
||||
&& !is_leftover
|
||||
{
|
||||
typesize
|
||||
} else {
|
||||
1
|
||||
};
|
||||
let neblock = bsize / nsplits;
|
||||
let mut pos = le32(src, HEADER + 4 * j)?;
|
||||
let tmp = &mut tmp[..bsize];
|
||||
for s in 0..nsplits {
|
||||
let clen = src
|
||||
.get(pos..)
|
||||
.and_then(|rest| rest.get(..4))
|
||||
.map(|b| u32::from_le_bytes(b.try_into().unwrap()) as usize)
|
||||
.ok_or_else(|| err("block offset out of range"))?;
|
||||
pos += 4;
|
||||
let stream = src
|
||||
.get(pos..pos.saturating_add(clen))
|
||||
.ok_or_else(|| err("stream runs past the frame"))?;
|
||||
let dst = &mut tmp[s * neblock..(s + 1) * neblock];
|
||||
if clen == neblock {
|
||||
dst.copy_from_slice(stream);
|
||||
} else {
|
||||
decode_stream(codec, stream, dst, &mut zstd)?;
|
||||
}
|
||||
pos += clen;
|
||||
}
|
||||
// `bsize` is a whole number of splits by construction (`nsplits` > 1
|
||||
// only for full blocks, and c-blosc sizes those in whole elements).
|
||||
if nsplits * neblock != bsize {
|
||||
return Err(err("block is not a whole number of streams"));
|
||||
}
|
||||
let dest = &mut out[j * blocksize..j * blocksize + bsize];
|
||||
unshuffle_block(flags, typesize, tmp, dest);
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Undo the frame's shuffle on one decoded block.
|
||||
fn unshuffle_block(flags: u8, typesize: usize, src: &[u8], dest: &mut [u8]) {
|
||||
let bsize = src.len();
|
||||
if flags & FLAG_SHUFFLE != 0 && typesize > 1 {
|
||||
let n = bsize / typesize;
|
||||
for i in 0..n {
|
||||
for b in 0..typesize {
|
||||
dest[i * typesize + b] = src[b * n + i];
|
||||
}
|
||||
}
|
||||
dest[n * typesize..].copy_from_slice(&src[n * typesize..]);
|
||||
} else if flags & FLAG_BITSHUFFLE != 0 && bsize >= typesize {
|
||||
let n = bsize / typesize;
|
||||
if n.is_multiple_of(8) {
|
||||
let body = n * typesize;
|
||||
bitunshuffle_block(&src[..body], &mut dest[..body], n, typesize);
|
||||
dest[body..].copy_from_slice(&src[body..]);
|
||||
} else {
|
||||
dest.copy_from_slice(src);
|
||||
}
|
||||
} else {
|
||||
dest.copy_from_slice(src);
|
||||
}
|
||||
}
|
||||
|
||||
/// BloscLZ decompression (c-blosc 1.21 `blosclz_decompress`): returns the
|
||||
/// number of bytes written, or 0 on malformed input — exactly as the C
|
||||
/// decoder, including stopping before a match that ends the stream, so a
|
||||
/// stream libblosc rejects is rejected here too.
|
||||
///
|
||||
/// Instructions: a control byte `ctrl`. Below 32, a literal run of
|
||||
/// `ctrl + 1` bytes. Otherwise a match: length `(ctrl >> 5) + 2`, extended
|
||||
/// by following bytes while they are 255 when the top three bits are all
|
||||
/// set; distance `((ctrl & 31) << 8) + next byte + 1`, or — when that byte
|
||||
/// is 255 and the high bits are 31 — a 16-bit big-endian distance plus 8192.
|
||||
/// The first instruction is always a literal.
|
||||
pub(crate) fn blosclz_decompress(input: &[u8], out: &mut [u8]) -> usize {
|
||||
const MAX_DISTANCE: usize = 8191;
|
||||
let limit = input.len();
|
||||
if limit == 0 {
|
||||
return 0;
|
||||
}
|
||||
let mut ip = 1usize;
|
||||
let mut op = 0usize;
|
||||
let mut ctrl = (input[0] & 31) as usize;
|
||||
loop {
|
||||
if ctrl >= 32 {
|
||||
let mut len = (ctrl >> 5) - 1;
|
||||
let ofs = (ctrl & 31) << 8;
|
||||
if len == 6 {
|
||||
loop {
|
||||
if ip + 1 >= limit {
|
||||
return 0;
|
||||
}
|
||||
let code = input[ip] as usize;
|
||||
ip += 1;
|
||||
len += code;
|
||||
if code != 255 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
} else if ip + 1 >= limit {
|
||||
return 0;
|
||||
}
|
||||
let code = input[ip] as usize;
|
||||
ip += 1;
|
||||
len += 3;
|
||||
// The copy source is `distance` bytes back.
|
||||
let mut distance = ofs + code + 1;
|
||||
if code == 255 && ofs == 31 << 8 {
|
||||
if ip + 1 >= limit {
|
||||
return 0;
|
||||
}
|
||||
let far = ((input[ip] as usize) << 8) + input[ip + 1] as usize;
|
||||
ip += 2;
|
||||
distance = far + MAX_DISTANCE + 1;
|
||||
}
|
||||
if op + len > out.len() {
|
||||
return 0;
|
||||
}
|
||||
if distance > op {
|
||||
return 0;
|
||||
}
|
||||
if ip >= limit {
|
||||
break;
|
||||
}
|
||||
ctrl = input[ip] as usize;
|
||||
ip += 1;
|
||||
let start = op - distance;
|
||||
if distance >= len {
|
||||
out.copy_within(start..start + len, op);
|
||||
} else {
|
||||
for k in 0..len {
|
||||
out[op + k] = out[start + k];
|
||||
}
|
||||
}
|
||||
op += len;
|
||||
} else {
|
||||
let run = ctrl + 1;
|
||||
if op + run > out.len() || ip + run > limit {
|
||||
return 0;
|
||||
}
|
||||
out[op..op + run].copy_from_slice(&input[ip..ip + run]);
|
||||
op += run;
|
||||
ip += run;
|
||||
if ip >= limit {
|
||||
break;
|
||||
}
|
||||
ctrl = input[ip] as usize;
|
||||
ip += 1;
|
||||
}
|
||||
}
|
||||
op
|
||||
}
|
||||
|
||||
/// The codec our encoder puts inside the frame.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) enum EncodeCodec {
|
||||
Lz4,
|
||||
Snappy,
|
||||
Zlib,
|
||||
Zstd,
|
||||
}
|
||||
|
||||
impl EncodeCodec {
|
||||
/// From the filter's `cd_values[6]` compressor code.
|
||||
fn from_cd(code: u32) -> Result<EncodeCodec, FormatError> {
|
||||
match code {
|
||||
1 | 2 => Ok(EncodeCodec::Lz4),
|
||||
3 => Ok(EncodeCodec::Snappy),
|
||||
4 => Ok(EncodeCodec::Zlib),
|
||||
5 => Ok(EncodeCodec::Zstd),
|
||||
0 => Err(FormatError::CompressionError(
|
||||
"blosc: clawhdf5 cannot write BloscLZ; choose lz4, snappy, zlib or zstd".into(),
|
||||
)),
|
||||
other => Err(FormatError::CompressionError(format!(
|
||||
"blosc: unknown compressor {other}"
|
||||
))),
|
||||
}
|
||||
}
|
||||
|
||||
fn flags(self) -> u8 {
|
||||
(match self {
|
||||
EncodeCodec::Lz4 => 1,
|
||||
EncodeCodec::Snappy => 2,
|
||||
EncodeCodec::Zlib => 3,
|
||||
EncodeCodec::Zstd => 4,
|
||||
}) << 5
|
||||
}
|
||||
|
||||
fn encode(self, data: &[u8], level: u32) -> Result<Vec<u8>, FormatError> {
|
||||
match self {
|
||||
EncodeCodec::Lz4 => Ok(lz4_flex::block::compress(data)),
|
||||
EncodeCodec::Snappy => snap::raw::Encoder::new()
|
||||
.compress_vec(data)
|
||||
.map_err(|e| FormatError::CompressionError(format!("blosc: snappy: {e}"))),
|
||||
EncodeCodec::Zlib => crate::filters::deflate_bounded(data, level.min(9))
|
||||
.map_err(|e| FormatError::CompressionError(format!("blosc: zlib: {e}"))),
|
||||
EncodeCodec::Zstd => Ok(crate::filters_bitshuffle::zstd_encode(data)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Block size our encoder uses: at most 256 KiB, a whole number of
|
||||
/// elements (and, for bit shuffle, of 8-element groups).
|
||||
fn encode_block_size(nbytes: usize, typesize: usize, bitshuffle: bool) -> usize {
|
||||
let unit = if bitshuffle { 8 * typesize } else { typesize };
|
||||
let target = (256 * 1024).min(nbytes);
|
||||
if target < unit {
|
||||
return nbytes.max(1);
|
||||
}
|
||||
target / unit * unit
|
||||
}
|
||||
|
||||
/// Encode a chunk as one Blosc 1 frame. `cd_values` as hdf5-blosc:
|
||||
/// `[2]` type size, `[4]` level (0 = store), `[5]` shuffle, `[6]` codec.
|
||||
pub(crate) fn blosc_encode(input: &[u8], ctx: &FilterContext<'_>) -> Result<Vec<u8>, FormatError> {
|
||||
let cd = ctx.client_data();
|
||||
let cerr = |m: &str| FormatError::CompressionError(format!("blosc: {m}"));
|
||||
let typesize = match cd.get(2) {
|
||||
Some(&t) if t != 0 => t as usize,
|
||||
_ => ctx.element_size.max(1),
|
||||
};
|
||||
// Blosc records the type size in one byte; c-blosc treats larger types
|
||||
// as bytes.
|
||||
let typesize = if typesize > 255 { 1 } else { typesize };
|
||||
let level = cd.get(4).copied().unwrap_or(5);
|
||||
let shuffle = cd.get(5).copied().unwrap_or(1);
|
||||
let codec = EncodeCodec::from_cd(cd.get(6).copied().unwrap_or(1))?;
|
||||
let nbytes = input.len();
|
||||
if nbytes > i32::MAX as usize - HEADER {
|
||||
return Err(cerr("chunk too large for a Blosc frame"));
|
||||
}
|
||||
let mut flags = codec.flags();
|
||||
match shuffle {
|
||||
0 => {}
|
||||
1 => flags |= FLAG_SHUFFLE,
|
||||
2 => flags |= FLAG_BITSHUFFLE,
|
||||
other => return Err(cerr(&format!("unknown shuffle mode {other}"))),
|
||||
}
|
||||
let blocksize = encode_block_size(nbytes, typesize, shuffle == 2);
|
||||
let header = |flags: u8, blocksize: usize, cbytes: usize| {
|
||||
let mut h = Vec::with_capacity(HEADER);
|
||||
h.extend_from_slice(&[2, 1, flags, typesize as u8]);
|
||||
h.extend_from_slice(&(nbytes as u32).to_le_bytes());
|
||||
h.extend_from_slice(&(blocksize as u32).to_le_bytes());
|
||||
h.extend_from_slice(&(cbytes as u32).to_le_bytes());
|
||||
h
|
||||
};
|
||||
let stored = || {
|
||||
let mut out = header(
|
||||
FLAG_MEMCPYED | (flags & !(FLAG_SHUFFLE | FLAG_BITSHUFFLE)),
|
||||
blocksize,
|
||||
nbytes + HEADER,
|
||||
);
|
||||
out.extend_from_slice(input);
|
||||
out
|
||||
};
|
||||
if level == 0 || nbytes == 0 {
|
||||
return Ok(stored());
|
||||
}
|
||||
|
||||
let nblocks = nbytes.div_ceil(blocksize);
|
||||
let leftover = nbytes % blocksize;
|
||||
let mut body = Vec::with_capacity(nbytes / 2);
|
||||
let mut starts = Vec::with_capacity(nblocks);
|
||||
let table_end = HEADER + 4 * nblocks;
|
||||
let mut shuffled = vec![0u8; blocksize];
|
||||
for j in 0..nblocks {
|
||||
let is_leftover = j == nblocks - 1 && leftover > 0;
|
||||
let bsize = if is_leftover { leftover } else { blocksize };
|
||||
let block = &input[j * blocksize..j * blocksize + bsize];
|
||||
let sh = &mut shuffled[..bsize];
|
||||
shuffle_block(flags, typesize, block, sh);
|
||||
starts.push(table_end + body.len());
|
||||
let nsplits = if typesize <= MAX_SPLITS
|
||||
&& bsize / typesize >= MIN_BUFFERSIZE
|
||||
&& !is_leftover
|
||||
&& bsize.is_multiple_of(typesize)
|
||||
{
|
||||
typesize
|
||||
} else {
|
||||
1
|
||||
};
|
||||
let neblock = bsize / nsplits;
|
||||
for s in 0..nsplits {
|
||||
let part = &sh[s * neblock..(s + 1) * neblock];
|
||||
let comp = codec.encode(part, level)?;
|
||||
if comp.len() < neblock {
|
||||
body.extend_from_slice(&(comp.len() as u32).to_le_bytes());
|
||||
body.extend_from_slice(&comp);
|
||||
} else {
|
||||
body.extend_from_slice(&(neblock as u32).to_le_bytes());
|
||||
body.extend_from_slice(part);
|
||||
}
|
||||
}
|
||||
if table_end + body.len() >= nbytes + HEADER {
|
||||
// Incompressible: store instead, as c-blosc does.
|
||||
return Ok(stored());
|
||||
}
|
||||
}
|
||||
// A split block must decode as split: the decoder infers splitting from
|
||||
// the same rule, which requires a whole number of elements per block.
|
||||
let cbytes = table_end + body.len();
|
||||
let mut out = header(flags, blocksize, cbytes);
|
||||
for s in starts {
|
||||
out.extend_from_slice(&(s as u32).to_le_bytes());
|
||||
}
|
||||
out.extend_from_slice(&body);
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Apply the frame's shuffle to one block (the inverse of
|
||||
/// [`unshuffle_block`]).
|
||||
fn shuffle_block(flags: u8, typesize: usize, src: &[u8], dest: &mut [u8]) {
|
||||
let bsize = src.len();
|
||||
if flags & FLAG_SHUFFLE != 0 && typesize > 1 {
|
||||
let n = bsize / typesize;
|
||||
for i in 0..n {
|
||||
for b in 0..typesize {
|
||||
dest[b * n + i] = src[i * typesize + b];
|
||||
}
|
||||
}
|
||||
dest[n * typesize..].copy_from_slice(&src[n * typesize..]);
|
||||
} else if flags & FLAG_BITSHUFFLE != 0 && bsize >= typesize {
|
||||
let n = bsize / typesize;
|
||||
if n.is_multiple_of(8) {
|
||||
let body = n * typesize;
|
||||
bitshuffle_block(&src[..body], &mut dest[..body], n, typesize);
|
||||
dest[body..].copy_from_slice(&src[body..]);
|
||||
} else {
|
||||
dest.copy_from_slice(src);
|
||||
}
|
||||
} else {
|
||||
dest.copy_from_slice(src);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::filter_pipeline::{FILTER_BLOSC, FilterDescription};
|
||||
|
||||
/// A blosclz stream: literal "abc", then a 9-byte match 3 back (a run
|
||||
/// of "abc"), then literal "Z".
|
||||
#[test]
|
||||
fn blosclz_decodes_literals_and_overlapping_matches() {
|
||||
// Match: length (ctrl >> 5) + 2 = 8, distance ofs + code + 1 = 3.
|
||||
let stream = [2, b'a', b'b', b'c', (6 << 5), 2, 0, b'Z'];
|
||||
let mut out = [0u8; 12];
|
||||
assert_eq!(blosclz_decompress(&stream, &mut out), 12);
|
||||
assert_eq!(&out, b"abcabcabcabZ");
|
||||
// A stream cut inside a match is malformed.
|
||||
let mut out = [0u8; 11];
|
||||
assert_eq!(blosclz_decompress(&stream[..6], &mut out), 0);
|
||||
// A match before the start of the output is malformed.
|
||||
assert_eq!(blosclz_decompress(&[0, b'a', 32, 5, 0, b'x'], &mut out), 0);
|
||||
}
|
||||
|
||||
fn desc(cd: Vec<u32>) -> FilterDescription {
|
||||
FilterDescription {
|
||||
filter_id: FILTER_BLOSC,
|
||||
name: None,
|
||||
flags: 0,
|
||||
client_data: cd,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn frame_round_trips_every_codec_and_shuffle() {
|
||||
for ts in [1usize, 2, 4, 8, 3, 32] {
|
||||
for n in [0usize, 5, 100, 1000, 70_000, 300_001] {
|
||||
if n * ts > 1 << 20 && ts > 1 {
|
||||
continue;
|
||||
}
|
||||
let data: Vec<u8> = (0..n * ts)
|
||||
.map(|i| ((i / ts) % 200) as u8 ^ (i % ts) as u8)
|
||||
.collect();
|
||||
for codec in [1u32, 3, 4, 5] {
|
||||
for shuffle in [0u32, 1, 2] {
|
||||
for level in [0u32, 5] {
|
||||
let f = desc(vec![2, 2, ts as u32, 0, level, shuffle, codec]);
|
||||
let ctx = FilterContext {
|
||||
filter: &f,
|
||||
element_size: ts,
|
||||
max_output: data.len(),
|
||||
};
|
||||
let enc = blosc_encode(&data, &ctx).unwrap();
|
||||
let dec = blosc_decode(&enc, &ctx).unwrap_or_else(|e| {
|
||||
panic!("ts={ts} n={n} codec={codec} shuffle={shuffle}: {e}")
|
||||
});
|
||||
assert!(
|
||||
dec == data,
|
||||
"ts={ts} n={n} codec={codec} shuffle={shuffle} level={level}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_bad_frames() {
|
||||
let data = vec![9u8; 50_000];
|
||||
let f = desc(vec![2, 2, 4, 0, 5, 1, 1]);
|
||||
let ctx = FilterContext {
|
||||
filter: &f,
|
||||
element_size: 4,
|
||||
max_output: data.len(),
|
||||
};
|
||||
let enc = blosc_encode(&data, &ctx).unwrap();
|
||||
assert!(blosc_decode(&enc[..enc.len() - 3], &ctx).is_err());
|
||||
let small = FilterContext {
|
||||
max_output: 49_999,
|
||||
..ctx
|
||||
};
|
||||
assert!(blosc_decode(&enc, &small).is_err());
|
||||
let mut v3 = enc.clone();
|
||||
v3[0] = 3;
|
||||
assert!(blosc_decode(&v3, &ctx).is_err());
|
||||
let f0 = desc(vec![2, 2, 4, 0, 5, 1, 0]);
|
||||
let ctx0 = FilterContext { filter: &f0, ..ctx };
|
||||
assert!(blosc_encode(&data, &ctx0).is_err());
|
||||
}
|
||||
|
||||
/// A frame that declares no data, for a chunk that has some.
|
||||
#[test]
|
||||
fn empty_frame_for_a_non_empty_chunk_is_an_error() {
|
||||
let mut frame = vec![2u8, 1, 0x20, 4];
|
||||
for v in [0u32, 64, 16] {
|
||||
frame.extend_from_slice(&v.to_le_bytes());
|
||||
}
|
||||
assert_eq!(blosc_decompress(&frame, 64).unwrap(), b"");
|
||||
let f = desc(vec![2, 2, 4, 64, 5, 1, 1]);
|
||||
let ctx = FilterContext {
|
||||
filter: &f,
|
||||
element_size: 4,
|
||||
max_output: 64,
|
||||
};
|
||||
assert!(blosc_decode(&frame, &ctx).is_err());
|
||||
}
|
||||
|
||||
/// A frame whose header claims a compressed size smaller than the
|
||||
/// header itself, not stored raw: an error, not an arithmetic overflow
|
||||
/// (it panicked in debug builds).
|
||||
#[test]
|
||||
fn frame_size_below_the_header_is_an_error() {
|
||||
let mut frame = vec![2u8, 1, 1 << 5, 4];
|
||||
for v in [64u32, 64, 8] {
|
||||
frame.extend_from_slice(&v.to_le_bytes());
|
||||
}
|
||||
frame.extend_from_slice(&[0; 40]);
|
||||
assert!(blosc_decompress(&frame, 1000).is_err());
|
||||
for cbytes in 0..16u32 {
|
||||
frame[12..16].copy_from_slice(&cbytes.to_le_bytes());
|
||||
assert!(blosc_decompress(&frame, 1000).is_err(), "cbytes={cbytes}");
|
||||
}
|
||||
}
|
||||
|
||||
/// A BloscLZ frame (our encoder cannot write one): a single block,
|
||||
/// one stream, no shuffle.
|
||||
fn blosclz_frame() -> Vec<u8> {
|
||||
let stream = [2, b'a', b'b', b'c', (6 << 5), 2, 0, b'Z'];
|
||||
let mut f = vec![2u8, 1, 0, 1];
|
||||
for v in [12u32, 12, (HEADER + 4 + 4 + stream.len()) as u32] {
|
||||
f.extend_from_slice(&v.to_le_bytes());
|
||||
}
|
||||
f.extend_from_slice(&((HEADER + 4) as u32).to_le_bytes());
|
||||
f.extend_from_slice(&(stream.len() as u32).to_le_bytes());
|
||||
f.extend_from_slice(&stream);
|
||||
f
|
||||
}
|
||||
|
||||
/// Random and mutated frames, every codec and shuffle: errors are fine,
|
||||
/// panics are not.
|
||||
#[test]
|
||||
fn fuzzed_frames_never_panic() {
|
||||
let limit = 6000;
|
||||
let data: Vec<u8> = (0..1500u32).flat_map(|i| (i / 5).to_le_bytes()).collect();
|
||||
let mut seeds = vec![blosclz_frame()];
|
||||
for codec in [1u32, 3, 4, 5] {
|
||||
for shuffle in [0u32, 1, 2] {
|
||||
for (ts, n) in [(4usize, data.len()), (4, 520), (1, 300), (2, 4)] {
|
||||
let f = desc(vec![2, 2, ts as u32, 0, 5, shuffle, codec]);
|
||||
let ctx = FilterContext {
|
||||
filter: &f,
|
||||
element_size: ts,
|
||||
max_output: n,
|
||||
};
|
||||
seeds.push(blosc_encode(&data[..n], &ctx).unwrap());
|
||||
}
|
||||
}
|
||||
}
|
||||
// Stored raw.
|
||||
let f = desc(vec![2, 2, 4, 0, 0, 1, 1]);
|
||||
let ctx = FilterContext {
|
||||
filter: &f,
|
||||
element_size: 4,
|
||||
max_output: 64,
|
||||
};
|
||||
seeds.push(blosc_encode(&data[..64], &ctx).unwrap());
|
||||
crate::test_fuzz::fuzz_decoder(0xb10, &seeds, 30_000, limit, |s| {
|
||||
blosc_decompress(s, limit)
|
||||
});
|
||||
}
|
||||
|
||||
/// BloscLZ streams on their own, random and mutated.
|
||||
#[test]
|
||||
fn fuzzed_blosclz_streams_never_panic() {
|
||||
let seed = blosclz_frame()[HEADER + 8..].to_vec();
|
||||
let mut out = [0u8; 64];
|
||||
crate::test_fuzz::fuzz_decoder(0xb11, &[seed], 30_000, 64, |s| {
|
||||
let n = blosclz_decompress(s, &mut out);
|
||||
if n == 0 {
|
||||
Err(err("malformed"))
|
||||
} else {
|
||||
Ok(out[..n].to_vec())
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
//! bzip2 (HDF5 filter 307, PyTables' `H5Zbzip2.c`, hdf5plugin's `BZip2`).
|
||||
//!
|
||||
//! The chunk is one bzip2 stream; `cd_values[0]` is the block size (1-9,
|
||||
//! the compression level). Decoded with the `bzip2` crate's default backend,
|
||||
//! `libbz2-rs-sys`, a pure-Rust port of libbzip2.
|
||||
|
||||
use crate::error::FormatError;
|
||||
use crate::filter_registry::FilterContext;
|
||||
|
||||
fn err(msg: &str) -> FormatError {
|
||||
FormatError::DecompressionError(format!("bzip2: {msg}"))
|
||||
}
|
||||
|
||||
/// Decode a bzip2-filtered chunk, refusing output beyond the chunk size.
|
||||
pub(crate) fn bzip2_decode(input: &[u8], ctx: &FilterContext<'_>) -> Result<Vec<u8>, FormatError> {
|
||||
use bzip2::{Decompress, Status};
|
||||
let limit = ctx.output_limit();
|
||||
let max_capacity = limit.saturating_add(1);
|
||||
let hint = if ctx.max_output != 0 {
|
||||
ctx.max_output
|
||||
} else {
|
||||
input.len().saturating_mul(4)
|
||||
};
|
||||
let mut out = Vec::new();
|
||||
out.try_reserve_exact(hint.clamp(1, max_capacity))
|
||||
.map_err(|_| err("cannot allocate the output buffer"))?;
|
||||
let mut dec = Decompress::new(false);
|
||||
loop {
|
||||
let (in_before, out_before) = (dec.total_in(), dec.total_out());
|
||||
let status = dec
|
||||
.decompress_vec(&input[in_before as usize..], &mut out)
|
||||
.map_err(|e| err(&e.to_string()))?;
|
||||
if out.len() > limit {
|
||||
return Err(err("output exceeds the chunk size"));
|
||||
}
|
||||
if status == Status::StreamEnd {
|
||||
return Ok(out);
|
||||
}
|
||||
if out.len() == out.capacity() {
|
||||
let grow = out
|
||||
.capacity()
|
||||
.min(max_capacity.saturating_sub(out.capacity()))
|
||||
.max(1);
|
||||
out.try_reserve_exact(grow)
|
||||
.map_err(|_| err("cannot allocate the output buffer"))?;
|
||||
} else if dec.total_in() as usize >= input.len()
|
||||
|| (dec.total_in(), dec.total_out()) == (in_before, out_before)
|
||||
{
|
||||
return Err(err("truncated stream"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Encode a chunk as one bzip2 stream at block size `cd_values[0]`
|
||||
/// (default 9, as hdf5plugin).
|
||||
pub(crate) fn bzip2_encode(input: &[u8], ctx: &FilterContext<'_>) -> Result<Vec<u8>, FormatError> {
|
||||
use bzip2::{Action, Compress, Compression, Status};
|
||||
let level = ctx.client_data().first().copied().unwrap_or(9).clamp(1, 9);
|
||||
let cerr = |m: String| FormatError::CompressionError(format!("bzip2: {m}"));
|
||||
let mut enc = Compress::new(Compression::new(level), 0);
|
||||
// bzip2's worst case is about 1% + 600 bytes over the input.
|
||||
let mut out = Vec::with_capacity(input.len() + input.len() / 100 + 600);
|
||||
loop {
|
||||
let consumed = enc.total_in() as usize;
|
||||
let status = enc
|
||||
.compress_vec(&input[consumed..], &mut out, Action::Finish)
|
||||
.map_err(|e| cerr(e.to_string()))?;
|
||||
if status == Status::StreamEnd {
|
||||
return Ok(out);
|
||||
}
|
||||
if out.len() == out.capacity() {
|
||||
out.reserve(out.capacity().max(4096));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::filter_pipeline::{FILTER_BZIP2, FilterDescription};
|
||||
|
||||
fn desc(level: u32) -> FilterDescription {
|
||||
FilterDescription {
|
||||
filter_id: FILTER_BZIP2,
|
||||
name: None,
|
||||
flags: 0,
|
||||
client_data: vec![level],
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn round_trips_and_bounds() {
|
||||
let data: Vec<u8> = (0..100_000u32)
|
||||
.flat_map(|i| (i % 777).to_le_bytes())
|
||||
.collect();
|
||||
for level in [1, 5, 9] {
|
||||
let f = desc(level);
|
||||
let ctx = FilterContext {
|
||||
filter: &f,
|
||||
element_size: 4,
|
||||
max_output: data.len(),
|
||||
};
|
||||
let enc = bzip2_encode(&data, &ctx).unwrap();
|
||||
assert!(enc.len() < data.len() / 4);
|
||||
assert_eq!(bzip2_decode(&enc, &ctx).unwrap(), data);
|
||||
// Truncated, and larger than the chunk: errors, not data.
|
||||
assert!(bzip2_decode(&enc[..enc.len() / 2], &ctx).is_err());
|
||||
let small = FilterContext {
|
||||
max_output: data.len() - 1,
|
||||
..ctx
|
||||
};
|
||||
assert!(bzip2_decode(&enc, &small).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
/// Random and mutated streams: errors are fine, panics are not.
|
||||
#[test]
|
||||
fn fuzzed_streams_never_panic() {
|
||||
let f = desc(9);
|
||||
let data: Vec<u8> = (0..4000u32).flat_map(|i| (i % 91).to_le_bytes()).collect();
|
||||
let ctx = FilterContext {
|
||||
filter: &f,
|
||||
element_size: 4,
|
||||
max_output: data.len(),
|
||||
};
|
||||
let seeds = vec![
|
||||
bzip2_encode(&data, &ctx).unwrap(),
|
||||
bzip2_encode(&data[..40], &ctx).unwrap(),
|
||||
];
|
||||
crate::test_fuzz::fuzz_decoder(0xb2, &seeds, 3_000, data.len(), |s| bzip2_decode(s, &ctx));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,260 @@
|
||||
//! LZF (HDF5 filter 32000) — h5py's built-in compression filter
|
||||
//! (`compression="lzf"`), in pure Rust.
|
||||
//!
|
||||
//! The chunk is one raw LZF stream (liblzf 3.x format, no header). The
|
||||
//! stream is a sequence of instructions, each starting with a control byte:
|
||||
//!
|
||||
//! * `000LLLLL` — a literal run: the next `L + 1` bytes (1..=32) are copied.
|
||||
//! * `LLLOOOOO [E] OOOOOOOO` — a back reference: copy `len + 2` bytes from
|
||||
//! `distance` bytes back, where `len` is the top three bits (1..=6), or
|
||||
//! `7 + E` when they are all ones, and `distance` is the 13-bit offset
|
||||
//! (high five bits in the control byte, low eight in the last byte) plus 1.
|
||||
//!
|
||||
//! h5py's filter (`lzf_filter.c`) records the chunk's size in bytes in
|
||||
//! `cd_values[2]` (slots 0 and 1 hold the filter and liblzf versions) and
|
||||
//! sizes its output buffer from it.
|
||||
|
||||
#[cfg(not(feature = "std"))]
|
||||
extern crate alloc;
|
||||
#[cfg(not(feature = "std"))]
|
||||
use alloc::{format, vec, vec::Vec};
|
||||
|
||||
use crate::error::FormatError;
|
||||
use crate::filter_registry::FilterContext;
|
||||
|
||||
/// `H5PY_FILTER_LZF_VERSION`, written to `cd_values[0]`.
|
||||
pub const LZF_FILTER_VERSION: u32 = 4;
|
||||
/// `LZF_VERSION` (liblzf 1.5), written to `cd_values[1]`.
|
||||
pub const LZF_API_VERSION: u32 = 0x0105;
|
||||
|
||||
const MAX_LITERAL: usize = 32;
|
||||
const MAX_OFFSET: usize = 1 << 13;
|
||||
const MAX_REF: usize = (1 << 8) + (1 << 3);
|
||||
const HASH_LOG: u32 = 14;
|
||||
|
||||
fn err(msg: &str) -> FormatError {
|
||||
FormatError::DecompressionError(format!("lzf: {msg}"))
|
||||
}
|
||||
|
||||
/// Decode an LZF-filtered chunk.
|
||||
pub(crate) fn lzf_decode(input: &[u8], ctx: &FilterContext<'_>) -> Result<Vec<u8>, FormatError> {
|
||||
let limit = ctx.output_limit();
|
||||
let hint = match ctx.client_data().get(2) {
|
||||
Some(&n) if n != 0 => n as usize,
|
||||
_ => input.len().saturating_mul(2),
|
||||
};
|
||||
lzf_decompress(input, hint.min(limit), limit)
|
||||
}
|
||||
|
||||
/// Decompress a raw LZF stream, refusing to produce more than `limit` bytes.
|
||||
pub fn lzf_decompress(
|
||||
input: &[u8],
|
||||
size_hint: usize,
|
||||
limit: usize,
|
||||
) -> Result<Vec<u8>, FormatError> {
|
||||
let mut out: Vec<u8> = Vec::new();
|
||||
out.try_reserve(size_hint)
|
||||
.map_err(|_| err("cannot allocate the output buffer"))?;
|
||||
let mut ip = 0usize;
|
||||
while ip < input.len() {
|
||||
let ctrl = input[ip] as usize;
|
||||
ip += 1;
|
||||
if ctrl < 32 {
|
||||
let run = ctrl + 1;
|
||||
let lit = input
|
||||
.get(ip..ip + run)
|
||||
.ok_or_else(|| err("literal run past the end of the input"))?;
|
||||
if out.len() + run > limit {
|
||||
return Err(err("output exceeds the chunk size"));
|
||||
}
|
||||
out.extend_from_slice(lit);
|
||||
ip += run;
|
||||
} else {
|
||||
let mut len = ctrl >> 5;
|
||||
if len == 7 {
|
||||
len += *input
|
||||
.get(ip)
|
||||
.ok_or_else(|| err("truncated back reference"))?
|
||||
as usize;
|
||||
ip += 1;
|
||||
}
|
||||
let low = *input
|
||||
.get(ip)
|
||||
.ok_or_else(|| err("truncated back reference"))? as usize;
|
||||
ip += 1;
|
||||
let distance = ((ctrl & 0x1f) << 8) + low + 1;
|
||||
let len = len + 2;
|
||||
if distance > out.len() {
|
||||
return Err(err("back reference before the start of the output"));
|
||||
}
|
||||
if out.len() + len > limit {
|
||||
return Err(err("output exceeds the chunk size"));
|
||||
}
|
||||
let start = out.len() - distance;
|
||||
if distance >= len {
|
||||
out.extend_from_within(start..start + len);
|
||||
} else {
|
||||
// Overlapping copy: repeats the last `distance` bytes.
|
||||
for k in 0..len {
|
||||
let b = out[start + k];
|
||||
out.push(b);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Encode a chunk with the LZF filter.
|
||||
pub(crate) fn lzf_encode(input: &[u8], _ctx: &FilterContext<'_>) -> Result<Vec<u8>, FormatError> {
|
||||
Ok(lzf_compress(input))
|
||||
}
|
||||
|
||||
fn hash3(b: &[u8]) -> usize {
|
||||
let v = (u32::from(b[0]) << 16) | (u32::from(b[1]) << 8) | u32::from(b[2]);
|
||||
(v.wrapping_mul(2_654_435_761) >> (32 - HASH_LOG)) as usize
|
||||
}
|
||||
|
||||
fn flush_literals(out: &mut Vec<u8>, lit: &[u8]) {
|
||||
for run in lit.chunks(MAX_LITERAL) {
|
||||
out.push((run.len() - 1) as u8);
|
||||
out.extend_from_slice(run);
|
||||
}
|
||||
}
|
||||
|
||||
/// Compress `input` into a raw LZF stream any liblzf decoder reads.
|
||||
///
|
||||
/// Incompressible input grows by one byte per 32. (h5py's own filter gives
|
||||
/// up on such a chunk and stores it unfiltered; storing the slightly larger
|
||||
/// stream is equally readable.)
|
||||
pub fn lzf_compress(input: &[u8]) -> Vec<u8> {
|
||||
let n = input.len();
|
||||
let mut out = Vec::with_capacity(n + n / MAX_LITERAL + 1);
|
||||
let mut table = vec![0u32; 1 << HASH_LOG];
|
||||
let mut lit_start = 0usize;
|
||||
let mut i = 0usize;
|
||||
while i + 2 < n {
|
||||
let h = hash3(&input[i..]);
|
||||
let cand = table[h] as usize;
|
||||
table[h] = (i + 1) as u32;
|
||||
if cand != 0 {
|
||||
let r = cand - 1;
|
||||
let distance = i - r;
|
||||
if distance <= MAX_OFFSET && input[r..r + 3] == input[i..i + 3] {
|
||||
let max_len = (n - i).min(MAX_REF);
|
||||
let mut len = 3;
|
||||
while len < max_len && input[r + len] == input[i + len] {
|
||||
len += 1;
|
||||
}
|
||||
flush_literals(&mut out, &input[lit_start..i]);
|
||||
let code = len - 2;
|
||||
let off = distance - 1;
|
||||
if code < 7 {
|
||||
out.push(((code << 5) | (off >> 8)) as u8);
|
||||
} else {
|
||||
out.push(((7 << 5) | (off >> 8)) as u8);
|
||||
out.push((code - 7) as u8);
|
||||
}
|
||||
out.push((off & 0xff) as u8);
|
||||
// Index the positions the match covered so later data can
|
||||
// refer back into it.
|
||||
let end = i + len;
|
||||
let mut j = i + 1;
|
||||
while j < end && j + 2 < n {
|
||||
table[hash3(&input[j..])] = (j + 1) as u32;
|
||||
j += 1;
|
||||
}
|
||||
i = end;
|
||||
lit_start = i;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
flush_literals(&mut out, &input[lit_start..]);
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn round_trip(data: &[u8]) {
|
||||
let c = lzf_compress(data);
|
||||
assert_eq!(lzf_decompress(&c, data.len(), data.len()).unwrap(), data);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn round_trips() {
|
||||
round_trip(b"");
|
||||
round_trip(b"a");
|
||||
round_trip(b"abcabcabcabcabcabcabcabcabcabcabcabc");
|
||||
round_trip(&[7u8; 10_000]);
|
||||
let noise: Vec<u8> = (0..70_000u32)
|
||||
.map(|i| (i.wrapping_mul(2_654_435_761) >> 13) as u8)
|
||||
.collect();
|
||||
round_trip(&noise);
|
||||
let ramp: Vec<u8> = (0..100_000u32)
|
||||
.flat_map(|i| (i % 1000).to_le_bytes())
|
||||
.collect();
|
||||
round_trip(&ramp);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compresses_repetitive_data() {
|
||||
let data = [42u8; 4096];
|
||||
assert!(lzf_compress(&data).len() < 100);
|
||||
}
|
||||
|
||||
/// The chunk h5py 3.16's bundled liblzf writes for
|
||||
/// `b"hello hello hello hello"` (read back with `read_direct_chunk`): a
|
||||
/// 7-byte literal, a 14-byte back reference 6 bytes back (extended
|
||||
/// length), and a 2-byte literal.
|
||||
#[test]
|
||||
fn decodes_liblzf_output() {
|
||||
let stream = b"\x06hello h\xe0\x05\x05\x01lo";
|
||||
assert_eq!(
|
||||
lzf_decompress(stream, 23, 23).unwrap(),
|
||||
b"hello hello hello hello"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_corrupt_streams() {
|
||||
// Back reference before the start.
|
||||
assert!(lzf_decompress(&[0x20, 0x00], 10, 10).is_err());
|
||||
// Literal run past the end.
|
||||
assert!(lzf_decompress(&[0x05, 1, 2], 10, 10).is_err());
|
||||
// Output over the limit.
|
||||
let c = lzf_compress(&[1u8; 100]);
|
||||
assert!(lzf_decompress(&c, 10, 99).is_err());
|
||||
}
|
||||
|
||||
/// Random and mutated streams: errors are fine, panics are not.
|
||||
#[test]
|
||||
fn fuzzed_streams_never_panic() {
|
||||
let seeds: Vec<Vec<u8>> = [
|
||||
b"hello hello hello hello".to_vec(),
|
||||
vec![7u8; 3000],
|
||||
(0..2000u32).flat_map(|i| (i % 37).to_le_bytes()).collect(),
|
||||
(0..500u32)
|
||||
.map(|i| (i.wrapping_mul(2_654_435_761) >> 13) as u8)
|
||||
.collect(),
|
||||
]
|
||||
.iter()
|
||||
.map(|d| lzf_compress(d))
|
||||
.collect();
|
||||
for limit in [0usize, 23, 4096, 8000] {
|
||||
crate::test_fuzz::fuzz_decoder(
|
||||
0x1f2 + limit as u64,
|
||||
&seeds[..1],
|
||||
5_000,
|
||||
limit.max(23),
|
||||
|s| lzf_decompress(s, limit, limit.max(23)),
|
||||
);
|
||||
}
|
||||
crate::test_fuzz::fuzz_decoder(0x1f3, &seeds, 20_000, 8000, |s| {
|
||||
lzf_decompress(s, 8000, 8000)
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -34,6 +34,7 @@ const SZ_NN_OPTION_MASK: u32 = 32;
|
||||
///
|
||||
/// The chunk is a 4-byte little-endian uncompressed size followed by the
|
||||
/// szlib stream.
|
||||
#[cfg_attr(not(feature = "szip"), allow(dead_code))]
|
||||
pub(crate) fn szip_decompress(
|
||||
_data: &[u8],
|
||||
_cd: &[u32],
|
||||
|
||||
@@ -43,6 +43,14 @@
|
||||
//! | `checksum` | yes | Jenkins lookup3 checksum validation |
|
||||
//! | `deflate` | yes | Deflate (gzip) compression via `flate2` |
|
||||
//! | `provenance` | yes | SHINES provenance — SHA-256 hashing & verification |
|
||||
//! | `lzf` | yes | LZF filter (32000), h5py's `compression="lzf"` |
|
||||
//! | `bitshuffle` | no | Bitshuffle filter (32008), none/LZ4/Zstandard |
|
||||
//! | `bzip2` | no | bzip2 filter (307) |
|
||||
//! | `blosc` | no | Blosc 1 filter (32001) |
|
||||
//! | `plugin-filters` | no | The four above |
|
||||
//!
|
||||
//! Filters are looked up by ID in [`filter_registry`], which also takes
|
||||
//! codecs registered at run time for other IDs.
|
||||
|
||||
#![cfg_attr(not(feature = "std"), no_std)]
|
||||
|
||||
@@ -71,7 +79,16 @@ pub mod extensible_array;
|
||||
pub mod file_writer;
|
||||
pub mod fill_value;
|
||||
pub mod filter_pipeline;
|
||||
pub mod filter_registry;
|
||||
pub mod filters;
|
||||
#[cfg(any(feature = "bitshuffle", feature = "blosc"))]
|
||||
mod filters_bitshuffle;
|
||||
#[cfg(feature = "blosc")]
|
||||
pub mod filters_blosc;
|
||||
#[cfg(feature = "bzip2")]
|
||||
mod filters_bzip2;
|
||||
#[cfg(feature = "lzf")]
|
||||
pub mod filters_lzf;
|
||||
mod filters_szip;
|
||||
pub mod fixed_array;
|
||||
pub mod float16;
|
||||
@@ -100,6 +117,16 @@ pub mod shared_message;
|
||||
pub mod signature;
|
||||
pub mod superblock;
|
||||
pub mod symbol_table;
|
||||
#[cfg(all(
|
||||
test,
|
||||
any(
|
||||
feature = "lzf",
|
||||
feature = "bitshuffle",
|
||||
feature = "bzip2",
|
||||
feature = "blosc"
|
||||
)
|
||||
))]
|
||||
mod test_fuzz;
|
||||
pub mod type_builders;
|
||||
pub mod vds;
|
||||
pub mod vl_data;
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
use crate::chunked_read::ChunkInfo;
|
||||
use crate::error::FormatError;
|
||||
use crate::filter_pipeline::FilterPipeline;
|
||||
use crate::filters::decompress_chunk_masked;
|
||||
use crate::filters::decompress_chunk_exact;
|
||||
use crate::lane_partition::{self, LaneStats, PartitionStats};
|
||||
|
||||
/// Threshold: only use parallel decompression when chunk count exceeds this.
|
||||
@@ -84,12 +84,13 @@ pub fn decompress_chunks_lane_partitioned(
|
||||
}
|
||||
let raw_chunk = &file_data[c_addr..c_addr + size];
|
||||
|
||||
let decompressed = decompress_chunk_masked(
|
||||
let decompressed = decompress_chunk_exact(
|
||||
raw_chunk,
|
||||
pipeline,
|
||||
chunk_total_bytes,
|
||||
element_size,
|
||||
chunk_info.filter_mask,
|
||||
&chunk_info.offsets,
|
||||
)?;
|
||||
|
||||
stats.chunks_processed += 1;
|
||||
@@ -160,12 +161,13 @@ pub fn decompress_chunks_parallel(
|
||||
}
|
||||
let raw_chunk = &file_data[c_addr..c_addr + size];
|
||||
|
||||
let decompressed = decompress_chunk_masked(
|
||||
let decompressed = decompress_chunk_exact(
|
||||
raw_chunk,
|
||||
pipeline,
|
||||
chunk_total_bytes,
|
||||
element_size,
|
||||
chunk_info.filter_mask,
|
||||
&chunk_info.offsets,
|
||||
)?;
|
||||
|
||||
Ok(DecompressedChunk {
|
||||
@@ -204,12 +206,13 @@ pub fn decompress_chunks_sequential(
|
||||
let raw_chunk = &file_data[c_addr..c_addr + size];
|
||||
|
||||
let decompressed = if let Some(pl) = pipeline {
|
||||
decompress_chunk_masked(
|
||||
decompress_chunk_exact(
|
||||
raw_chunk,
|
||||
pl,
|
||||
chunk_total_bytes,
|
||||
element_size,
|
||||
chunk_info.filter_mask,
|
||||
&chunk_info.offsets,
|
||||
)?
|
||||
} else {
|
||||
raw_chunk.to_vec()
|
||||
@@ -218,3 +221,60 @@ pub fn decompress_chunks_sequential(
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::filter_pipeline::{FILTER_SHUFFLE, FilterDescription};
|
||||
|
||||
/// Eight shuffled 32-byte chunks; chunk 5 is stored short when `short`.
|
||||
fn chunks(short: bool) -> (Vec<u8>, Vec<ChunkInfo>) {
|
||||
let mut file = Vec::new();
|
||||
let mut infos = Vec::new();
|
||||
for i in 0..8u64 {
|
||||
let len = if short && i == 5 { 16 } else { 32 };
|
||||
infos.push(ChunkInfo {
|
||||
chunk_size: len as u32,
|
||||
filter_mask: 0,
|
||||
offsets: vec![i * 8],
|
||||
address: file.len() as u64,
|
||||
});
|
||||
file.extend(core::iter::repeat_n(i as u8, len));
|
||||
}
|
||||
(file, infos)
|
||||
}
|
||||
|
||||
/// Every parallel decoder refuses a chunk that decodes short, naming it.
|
||||
#[test]
|
||||
fn short_decoded_chunk_is_an_error() {
|
||||
let pipeline = FilterPipeline {
|
||||
version: 2,
|
||||
filters: vec![FilterDescription {
|
||||
filter_id: FILTER_SHUFFLE,
|
||||
name: None,
|
||||
flags: 0,
|
||||
client_data: vec![4],
|
||||
}],
|
||||
};
|
||||
let (file, good) = chunks(false);
|
||||
assert_eq!(
|
||||
decompress_chunks_parallel(&file, &good, &pipeline, 32, 4).unwrap()[5],
|
||||
[5u8; 32]
|
||||
);
|
||||
let (file, bad) = chunks(true);
|
||||
let errs = [
|
||||
decompress_chunks_lane_partitioned(&file, &bad, &pipeline, 32, 4, 1, Some(3))
|
||||
.map(|_| ())
|
||||
.unwrap_err(),
|
||||
decompress_chunks_parallel(&file, &bad, &pipeline, 32, 4)
|
||||
.map(|_| ())
|
||||
.unwrap_err(),
|
||||
decompress_chunks_sequential(&file, &bad, Some(&pipeline), 32, 4)
|
||||
.map(|_| ())
|
||||
.unwrap_err(),
|
||||
];
|
||||
for e in errs {
|
||||
assert!(e.to_string().contains("[40]"), "{e}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -22,7 +22,7 @@ use crate::data_read::extract_selection_from_buffer;
|
||||
use crate::dataspace::Dataspace;
|
||||
use crate::error::FormatError;
|
||||
use crate::filter_pipeline::FilterPipeline;
|
||||
use crate::filters::{all_filters_skipped, decompress_chunk_masked};
|
||||
use crate::filters::{all_filters_skipped, decompress_chunk_exact};
|
||||
use crate::selection::Selection;
|
||||
|
||||
/// The smallest axis-aligned box containing every selected element, as
|
||||
@@ -330,12 +330,13 @@ pub fn read_selection(
|
||||
let decoded;
|
||||
let data: &[u8] = match pipeline {
|
||||
Some(pl) if !all_filters_skipped(pl, chunk.filter_mask) => {
|
||||
decoded = decompress_chunk_masked(
|
||||
decoded = decompress_chunk_exact(
|
||||
raw,
|
||||
pl,
|
||||
chunk_bytes,
|
||||
elem_size as u32,
|
||||
chunk.filter_mask,
|
||||
&chunk.offsets[..rank],
|
||||
)?;
|
||||
&decoded
|
||||
}
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
//! Mutation fuzzing for the filter decoders (tests only).
|
||||
//!
|
||||
//! A decoder fed a random or mutated frame may fail, but must not panic —
|
||||
//! tests build with overflow checks and debug assertions, so an unchecked
|
||||
//! subtraction, multiplication or shift on a header field, or an
|
||||
//! out-of-range slice, fails the test — and must not return more than its
|
||||
//! output limit.
|
||||
|
||||
#[cfg(not(feature = "std"))]
|
||||
extern crate alloc;
|
||||
#[cfg(not(feature = "std"))]
|
||||
use alloc::vec::Vec;
|
||||
|
||||
use crate::error::FormatError;
|
||||
|
||||
/// xorshift64*: deterministic, so a failure reproduces.
|
||||
pub(crate) struct Rng(u64);
|
||||
|
||||
impl Rng {
|
||||
pub(crate) fn new(seed: u64) -> Rng {
|
||||
Rng(seed.max(1))
|
||||
}
|
||||
|
||||
pub(crate) fn next_u64(&mut self) -> u64 {
|
||||
let mut x = self.0;
|
||||
x ^= x >> 12;
|
||||
x ^= x << 25;
|
||||
x ^= x >> 27;
|
||||
self.0 = x;
|
||||
x.wrapping_mul(0x2545_F491_4F6C_DD1D)
|
||||
}
|
||||
|
||||
/// Uniform in `0..n` (`n` > 0).
|
||||
pub(crate) fn below(&mut self, n: usize) -> usize {
|
||||
(self.next_u64() % n as u64) as usize
|
||||
}
|
||||
|
||||
pub(crate) fn bytes(&mut self, n: usize) -> Vec<u8> {
|
||||
(0..n).map(|_| self.next_u64() as u8).collect()
|
||||
}
|
||||
|
||||
/// A u32 that tends to hit edge cases in size and offset fields.
|
||||
fn interesting_u32(&mut self, len: usize) -> u32 {
|
||||
match self.below(10) {
|
||||
0 => 0,
|
||||
1 => 1,
|
||||
2 => self.below(20) as u32,
|
||||
3 => 15 + self.below(3) as u32,
|
||||
4 => u32::MAX - self.below(16) as u32,
|
||||
5 => 1 << self.below(32),
|
||||
6 => (len as u32)
|
||||
.wrapping_add(self.below(9) as u32)
|
||||
.wrapping_sub(4),
|
||||
7 => i32::MAX as u32,
|
||||
_ => self.next_u64() as u32,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One to four random edits of `seed`.
|
||||
pub(crate) fn mutate(rng: &mut Rng, seed: &[u8]) -> Vec<u8> {
|
||||
let mut v = seed.to_vec();
|
||||
for _ in 0..1 + rng.below(4) {
|
||||
let len = v.len();
|
||||
match rng.below(9) {
|
||||
0 if len > 0 => {
|
||||
let i = rng.below(len);
|
||||
v[i] ^= 1 << rng.below(8);
|
||||
}
|
||||
1 if len > 0 => {
|
||||
let i = rng.below(len);
|
||||
v[i] = rng.next_u64() as u8;
|
||||
}
|
||||
2 if len > 0 => {
|
||||
let i = rng.below(len);
|
||||
v[i] = [0, 0xff, 0x7f, 0x80, 0x20, 0x1f][rng.below(6)];
|
||||
}
|
||||
// A size or offset field: little- or big-endian, anywhere, but
|
||||
// most often in the first 32 bytes where headers live.
|
||||
3 | 4 if len >= 4 => {
|
||||
let span = if rng.below(2) == 0 { len.min(32) } else { len };
|
||||
let i = rng.below(span - 3);
|
||||
let x = rng.interesting_u32(len);
|
||||
let b = if rng.below(2) == 0 {
|
||||
x.to_le_bytes()
|
||||
} else {
|
||||
x.to_be_bytes()
|
||||
};
|
||||
v[i..i + 4].copy_from_slice(&b);
|
||||
}
|
||||
5 if len > 0 => v.truncate(rng.below(len)),
|
||||
6 => {
|
||||
let n = 1 + rng.below(64);
|
||||
let extra = rng.bytes(n);
|
||||
v.extend_from_slice(&extra);
|
||||
}
|
||||
7 if len > 1 => {
|
||||
let a = rng.below(len);
|
||||
let b = a + rng.below(len - a);
|
||||
let copy = v[a..b].to_vec();
|
||||
let at = rng.below(len);
|
||||
v.splice(at..at, copy);
|
||||
}
|
||||
_ if len > 0 => {
|
||||
let i = rng.below(len);
|
||||
v[i] = v[i].wrapping_add(1 + rng.below(3) as u8);
|
||||
}
|
||||
_ => v.push(rng.next_u64() as u8),
|
||||
}
|
||||
}
|
||||
v
|
||||
}
|
||||
|
||||
/// Feed `iters` inputs to `decode`: mostly mutations of `seeds`, some pure
|
||||
/// noise and some truncated seeds. Asserts only "no panic, output within
|
||||
/// `limit`".
|
||||
pub(crate) fn fuzz_decoder(
|
||||
seed: u64,
|
||||
seeds: &[Vec<u8>],
|
||||
iters: usize,
|
||||
limit: usize,
|
||||
mut decode: impl FnMut(&[u8]) -> Result<Vec<u8>, FormatError>,
|
||||
) {
|
||||
assert!(!seeds.is_empty());
|
||||
let mut rng = Rng::new(seed);
|
||||
for s in seeds {
|
||||
// The seeds themselves must be valid, or the fuzz explores nothing.
|
||||
decode(s).expect("seed frame must decode");
|
||||
}
|
||||
for _ in 0..iters {
|
||||
let input = match rng.below(16) {
|
||||
0 => {
|
||||
let n = rng.below(96);
|
||||
rng.bytes(n)
|
||||
}
|
||||
1 => {
|
||||
let s = &seeds[rng.below(seeds.len())];
|
||||
s[..rng.below(s.len() + 1)].to_vec()
|
||||
}
|
||||
_ => {
|
||||
let s = &seeds[rng.below(seeds.len())];
|
||||
mutate(&mut rng, s)
|
||||
}
|
||||
};
|
||||
if let Ok(out) = decode(&input) {
|
||||
assert!(out.len() <= limit, "decoded {} > limit {limit}", out.len());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -731,6 +731,61 @@ impl DatasetBuilder {
|
||||
self
|
||||
}
|
||||
|
||||
/// Compress with a plugin filter ([`PluginFilter`]), in the format the
|
||||
/// libhdf5 plugin reads (h5py, hdf5plugin). Implies chunked storage.
|
||||
/// Each filter needs its cargo feature (`lzf`, ...); writing fails with
|
||||
/// `UnsupportedFilter` without it.
|
||||
///
|
||||
/// [`PluginFilter`]: crate::chunked_write::PluginFilter
|
||||
pub fn with_plugin_filter(&mut self, filter: crate::chunked_write::PluginFilter) -> &mut Self {
|
||||
self.chunk_options.plugin = Some(filter);
|
||||
self
|
||||
}
|
||||
|
||||
/// Enable LZF compression (filter 32000) — h5py's built-in
|
||||
/// `compression="lzf"`. Implies chunked storage; shuffle is applied
|
||||
/// first unless `.without_shuffle()`. Requires the `lzf` cargo feature.
|
||||
pub fn with_lzf(&mut self) -> &mut Self {
|
||||
self.with_plugin_filter(crate::chunked_write::PluginFilter::Lzf)
|
||||
}
|
||||
|
||||
/// Enable bitshuffle (filter 32008) with `compression` after the bit
|
||||
/// transpose, in bitshuffle's default block size. Implies chunked
|
||||
/// storage; no byte shuffle is added. Requires the `bitshuffle` cargo
|
||||
/// feature.
|
||||
pub fn with_bitshuffle(
|
||||
&mut self,
|
||||
compression: crate::chunked_write::BitshuffleCompression,
|
||||
) -> &mut Self {
|
||||
self.with_plugin_filter(crate::chunked_write::PluginFilter::Bitshuffle {
|
||||
block_size: 0,
|
||||
compression,
|
||||
})
|
||||
}
|
||||
|
||||
/// Enable bzip2 (filter 307) at block size `level` (1-9). Implies
|
||||
/// chunked storage; shuffle is applied first unless
|
||||
/// `.without_shuffle()`. Requires the `bzip2` cargo feature.
|
||||
pub fn with_bzip2(&mut self, level: u32) -> &mut Self {
|
||||
self.with_plugin_filter(crate::chunked_write::PluginFilter::Bzip2 { level })
|
||||
}
|
||||
|
||||
/// Enable Blosc (filter 32001) with `codec` at `level` (0-9) after
|
||||
/// `shuffle`. Implies chunked storage; no extra HDF5 shuffle is added.
|
||||
/// Requires the `blosc` cargo feature.
|
||||
pub fn with_blosc(
|
||||
&mut self,
|
||||
codec: crate::chunked_write::BloscCodec,
|
||||
level: u32,
|
||||
shuffle: crate::chunked_write::BloscShuffle,
|
||||
) -> &mut Self {
|
||||
self.with_plugin_filter(crate::chunked_write::PluginFilter::Blosc {
|
||||
codec,
|
||||
level,
|
||||
shuffle,
|
||||
})
|
||||
}
|
||||
|
||||
/// Enable Pcodec lossless numerical compression (private clawhdf5 filter
|
||||
/// ID 480).
|
||||
///
|
||||
|
||||
@@ -31,7 +31,7 @@ name = "parallel_bench"
|
||||
harness = false
|
||||
|
||||
[features]
|
||||
default = ["mmap", "provenance"]
|
||||
default = ["mmap", "provenance", "lzf"]
|
||||
mmap = ["clawhdf5-io/mmap"]
|
||||
parallel = ["clawhdf5-format/parallel", "rayon"]
|
||||
# zlib-ng (C, needs cmake) instead of the default pure-Rust zlib-rs.
|
||||
@@ -41,6 +41,14 @@ zstd = ["clawhdf5-format/zstd"]
|
||||
blake3_hash = ["clawhdf5-format/blake3_hash"]
|
||||
lz4 = ["clawhdf5-format/lz4"]
|
||||
pcodec = ["clawhdf5-format/pcodec"]
|
||||
# Plugin filters, pure Rust (no C). LZF (32000) is h5py's built-in
|
||||
# compression; it has no dependencies, so it is on by default.
|
||||
lzf = ["clawhdf5-format/lzf"]
|
||||
bitshuffle = ["clawhdf5-format/bitshuffle"]
|
||||
bzip2 = ["clawhdf5-format/bzip2"]
|
||||
blosc = ["clawhdf5-format/blosc"]
|
||||
# Every plugin filter.
|
||||
plugin-filters = ["lzf", "bitshuffle", "bzip2", "blosc"]
|
||||
# Dataset::verify_provenance() — recompute a dataset's SHA-256 and compare
|
||||
# against its stored _provenance_sha256 attribute. On by default, matching
|
||||
# clawhdf5-format's own default-on `provenance` feature.
|
||||
|
||||
@@ -381,3 +381,77 @@ with h5py.File("{p}", "r") as f:
|
||||
let fa = file.dataset("fa").unwrap().read_i32().unwrap();
|
||||
assert!(fa.iter().copied().eq(0..3 * 70000), "fa");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Chunks that decode short
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// HDF5 stores every chunk at the full chunk size, so a chunk whose filters
|
||||
/// decode to fewer bytes is corrupt. libhdf5 returns the rest of such a
|
||||
/// chunk uninitialised (or, for filters that check, fails); clawhdf5 padded
|
||||
/// it with zeros and returned it as data. Every read path must fail,
|
||||
/// naming the chunk, and chunks that decode fully must still read.
|
||||
#[test]
|
||||
fn h5py_short_decoded_chunk_is_an_error() {
|
||||
skip_if_no_python!();
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("short.h5");
|
||||
let p = path.display().to_string();
|
||||
run_python(&format!(
|
||||
r#"
|
||||
import h5py, numpy as np, zlib
|
||||
with h5py.File("{p}", "w") as f:
|
||||
# 1-D, 4 gzip chunks of 8 i4; chunk 2 (offset 16) inflates to 16 bytes.
|
||||
ds = f.create_dataset("line", shape=(32,), chunks=(8,), dtype="<i4", compression="gzip")
|
||||
ds[...] = np.arange(32, dtype="<i4") + 1
|
||||
ds.id.write_direct_chunk((16,), zlib.compress(np.arange(4, dtype="<i4").tobytes()))
|
||||
# 2-D with a partial edge chunk; the short chunk is the edge one (8, 4).
|
||||
g = f.create_dataset("grid", shape=(10, 7), chunks=(4, 4), dtype="<f8", compression="gzip")
|
||||
g[...] = np.arange(70, dtype="<f8").reshape(10, 7) + 1
|
||||
g.id.write_direct_chunk((8, 4), zlib.compress(np.ones(3, dtype="<f8").tobytes()))
|
||||
# 40 chunks (enough for the parallel decoder), one short, shuffle + gzip.
|
||||
m = f.create_dataset("many", shape=(320,), chunks=(8,), dtype="<i4",
|
||||
compression="gzip", shuffle=True)
|
||||
m[...] = np.arange(320, dtype="<i4") + 1
|
||||
m.id.write_direct_chunk((200,), zlib.compress(bytes(31)))
|
||||
"#
|
||||
));
|
||||
|
||||
let hyperslab = |start: u64, count: u64| Selection::Hyperslab {
|
||||
start: vec![start],
|
||||
stride: vec![1],
|
||||
count: vec![1],
|
||||
block: vec![count],
|
||||
};
|
||||
let file = File::open(&path).unwrap();
|
||||
for (name, coords) in [("line", "[16"), ("grid", "[8, 4"), ("many", "[200")] {
|
||||
let ds = file.dataset(name).unwrap();
|
||||
let full = || {
|
||||
if name == "grid" {
|
||||
ds.read_f64().map(|_| ())
|
||||
} else {
|
||||
ds.read_i32().map(|_| ())
|
||||
}
|
||||
};
|
||||
let err = full().expect_err(name).to_string();
|
||||
assert!(err.contains(coords), "{name}: {err}");
|
||||
// Cached reads decode the same way; a second read fails too.
|
||||
assert!(full().is_err(), "{name}");
|
||||
assert!(ds.read_selection(&Selection::All).is_err(), "{name}");
|
||||
}
|
||||
let line = file.dataset("line").unwrap();
|
||||
assert!(line.read_selection(&hyperslab(14, 4)).is_err());
|
||||
// A selection that avoids the short chunk still reads.
|
||||
let want: Vec<u8> = (1..=16i32).flat_map(i32::to_le_bytes).collect();
|
||||
assert_eq!(line.read_selection(&hyperslab(0, 16)).unwrap(), want);
|
||||
let many = file.dataset("many").unwrap();
|
||||
assert!(many.read_selection(&hyperslab(190, 20)).is_err());
|
||||
|
||||
// The memory-mapped and lazy readers.
|
||||
let mm = clawhdf5::MmapFile::open(&path).unwrap();
|
||||
assert!(mm.dataset("line").unwrap().read_i32().is_err());
|
||||
assert!(mm.dataset("many").unwrap().read_i32().is_err());
|
||||
let lazy = clawhdf5::LazyFile::open_mmap(&path).unwrap();
|
||||
assert!(lazy.dataset("line").unwrap().read_i32().is_err());
|
||||
assert!(lazy.dataset("grid").unwrap().read_f64().is_err());
|
||||
}
|
||||
|
||||
@@ -0,0 +1,469 @@
|
||||
//! Plugin filters (LZF, bitshuffle, bzip2, blosc) against libhdf5.
|
||||
//!
|
||||
//! Read direction: h5py (with hdf5plugin for everything but LZF, which h5py
|
||||
//! ships) writes each filter over a matrix of dtypes (1-8 bytes, both byte
|
||||
//! orders), 1-3 dimensional shapes whose chunks do not divide them (so the
|
||||
//! edge chunks are partial), compressible and incompressible data, and the
|
||||
//! filter's own options; every dataset has an unfiltered twin, and clawhdf5
|
||||
//! must read the filtered one byte for byte equal to it.
|
||||
//!
|
||||
//! Write direction: clawhdf5 writes with its encoder, and h5py must read the
|
||||
//! values back.
|
||||
//!
|
||||
//! Skipped when python3 with h5py (and hdf5plugin) is unavailable, unless
|
||||
//! `CLAWHDF5_REQUIRE_INTEROP=1`.
|
||||
#![allow(dead_code)]
|
||||
|
||||
use std::process::Command;
|
||||
|
||||
use clawhdf5::File;
|
||||
use clawhdf5_format::selection::Selection;
|
||||
|
||||
fn python() -> String {
|
||||
std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
|
||||
}
|
||||
|
||||
fn interop_required() -> bool {
|
||||
std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
|
||||
}
|
||||
|
||||
fn python_has(modules: &str) -> bool {
|
||||
Command::new(python())
|
||||
.args(["-c", &format!("import {modules}")])
|
||||
.output()
|
||||
.map(|o| o.status.success())
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
/// Whether the interop test can run; panics instead of skipping when
|
||||
/// `CLAWHDF5_REQUIRE_INTEROP=1`.
|
||||
fn have_python(modules: &str) -> bool {
|
||||
if python_has(modules) {
|
||||
return true;
|
||||
}
|
||||
assert!(
|
||||
!interop_required(),
|
||||
"CLAWHDF5_REQUIRE_INTEROP=1 but python3 with {modules} is not available"
|
||||
);
|
||||
eprintln!("SKIP: python3 with {modules} not available");
|
||||
false
|
||||
}
|
||||
|
||||
fn run_python(script: &str, args: &[&str]) -> String {
|
||||
let output = Command::new(python())
|
||||
.arg("-c")
|
||||
.arg(script)
|
||||
.args(args)
|
||||
.output()
|
||||
.expect("failed to run python");
|
||||
if !output.status.success() {
|
||||
panic!(
|
||||
"Python script failed:\nSTDOUT: {}\nSTDERR: {}",
|
||||
String::from_utf8_lossy(&output.stdout),
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
}
|
||||
String::from_utf8_lossy(&output.stdout).trim().to_string()
|
||||
}
|
||||
|
||||
/// Writes `f{i}` (filtered) and `r{i}` (unfiltered twin) for every filter
|
||||
/// setting in `FILTERS` × every case; prints the number of pairs.
|
||||
const GENERATE: &str = r#"
|
||||
import sys
|
||||
import numpy as np, h5py
|
||||
try:
|
||||
import hdf5plugin
|
||||
except ImportError:
|
||||
hdf5plugin = None
|
||||
path = sys.argv[1]
|
||||
FILTERS = eval('(' + sys.argv[2] + ')')
|
||||
cases = [
|
||||
('<u1', (1000,), (128,), 'ramp'),
|
||||
('<i2', (37, 53), (10, 16), 'ramp'),
|
||||
('<i4', (2000,), (300,), 'ramp'),
|
||||
('>i4', (37, 53), (8, 8), 'ramp'),
|
||||
('<f4', (5, 6, 7), (2, 3, 4), 'ramp'),
|
||||
('<f8', (1500,), (512,), 'ramp'),
|
||||
('<i8', (33, 17), (33, 17), 'ramp'),
|
||||
('<u2', (4097,), (4097,), 'ramp'),
|
||||
('<f8', (3000,), (1000,), 'noise'),
|
||||
('<u1', (5000,), (5000,), 'noise'),
|
||||
('<i4', (1, 1), (1, 1), 'ramp'),
|
||||
('<f8', (100000,), (40000,), 'ramp'),
|
||||
]
|
||||
rng = np.random.default_rng(7)
|
||||
i = 0
|
||||
with h5py.File(path, 'w') as f:
|
||||
for label, kw in FILTERS:
|
||||
for dt, shape, chunks, kind in cases:
|
||||
n = int(np.prod(shape))
|
||||
if kind == 'noise':
|
||||
raw = rng.integers(0, 256, n * np.dtype(dt).itemsize, dtype=np.uint8)
|
||||
data = raw.view(dt)
|
||||
if np.dtype(dt).kind == 'f':
|
||||
data = np.nan_to_num(data)
|
||||
else:
|
||||
base = (np.arange(n) * 3) % 251 + rng.integers(0, 4, n)
|
||||
data = (base / 7).astype(dt) if np.dtype(dt).kind == 'f' else base.astype(dt)
|
||||
data = data.reshape(shape)
|
||||
f.create_dataset(f'f{i}', data=data, chunks=chunks, **kw)
|
||||
f.create_dataset(f'r{i}', data=data)
|
||||
f[f'f{i}'].attrs['case'] = f'{label} {dt} {shape} chunks={chunks} {kind}'
|
||||
i += 1
|
||||
print(i)
|
||||
"#;
|
||||
|
||||
/// Have h5py write every filter setting in `filters` (a Python list of
|
||||
/// `(label, create_dataset kwargs)`), then check that clawhdf5 reads each
|
||||
/// filtered dataset exactly as its unfiltered twin.
|
||||
fn check_h5py_written(tag: &str, filters: &str) {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join(format!("{tag}.h5"));
|
||||
let n: usize = run_python(GENERATE, &[path.to_str().unwrap(), filters])
|
||||
.parse()
|
||||
.unwrap();
|
||||
assert!(n > 0);
|
||||
let file = File::open(&path).unwrap();
|
||||
for i in 0..n {
|
||||
let filtered = file.dataset(&format!("f{i}")).unwrap();
|
||||
let case = format!("{:?}", filtered.attrs().unwrap().get("case"));
|
||||
let got = filtered
|
||||
.read_selection(&Selection::All)
|
||||
.unwrap_or_else(|e| panic!("{tag} f{i} {case}: {e}"));
|
||||
let want = file
|
||||
.dataset(&format!("r{i}"))
|
||||
.unwrap()
|
||||
.read_selection(&Selection::All)
|
||||
.unwrap();
|
||||
assert!(got == want, "{tag} f{i} {case}: data differs");
|
||||
}
|
||||
}
|
||||
|
||||
/// Values the write-direction tests store: row-major, compressible with some
|
||||
/// variation.
|
||||
fn ramp_i32(n: usize) -> Vec<i32> {
|
||||
(0..n).map(|i| ((i * 3) % 251) as i32 - 60).collect()
|
||||
}
|
||||
fn ramp_f64(n: usize) -> Vec<f64> {
|
||||
(0..n).map(|i| ((i * 3) % 251) as f64 / 7.0).collect()
|
||||
}
|
||||
fn ramp_u8(n: usize) -> Vec<u8> {
|
||||
(0..n).map(|i| ((i * 7) % 256) as u8).collect()
|
||||
}
|
||||
|
||||
/// h5py checks the datasets `write_ours` wrote against the same ramps.
|
||||
const VERIFY: &str = r#"
|
||||
import sys
|
||||
import numpy as np, h5py
|
||||
try:
|
||||
import hdf5plugin
|
||||
except ImportError:
|
||||
pass
|
||||
bad = []
|
||||
with h5py.File(sys.argv[1], 'r') as f:
|
||||
for name in f:
|
||||
ds = f[name]
|
||||
n = ds.size
|
||||
k = np.arange(n)
|
||||
if ds.dtype == np.int32:
|
||||
want = ((k * 3) % 251 - 60).astype(np.int32)
|
||||
elif ds.dtype == np.float64:
|
||||
want = ((k * 3) % 251) / 7.0
|
||||
else:
|
||||
want = ((k * 7) % 256).astype(np.uint8)
|
||||
got = ds[()].reshape(-1)
|
||||
if not np.array_equal(got, want):
|
||||
bad.append(name)
|
||||
# The dataset really is filtered by the plugin under test.
|
||||
if int(sys.argv[2]) not in [int(x) for x in ds._filters.keys() if x.isdigit()] \
|
||||
and sys.argv[3] not in ds._filters:
|
||||
bad.append(name + ':filter-missing:' + repr(ds._filters))
|
||||
print('OK' if not bad else 'BAD ' + ' '.join(bad))
|
||||
"#;
|
||||
|
||||
/// Write i32/f64/u8 datasets (1-D and 2-D, partial edge chunks) with
|
||||
/// `configure` applying the filter, then have h5py read them back.
|
||||
fn check_ours_read_by_h5py(
|
||||
tag: &str,
|
||||
filter_id: u16,
|
||||
filter_name: &str,
|
||||
configure: impl Fn(&mut clawhdf5_format::type_builders::DatasetBuilder),
|
||||
) {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join(format!("{tag}_ours.h5"));
|
||||
let mut fb = clawhdf5::FileBuilder::new();
|
||||
{
|
||||
let ds = fb.create_dataset("i32_1d");
|
||||
ds.with_i32_data(&ramp_i32(10_000)).with_chunks(&[3000]);
|
||||
configure(ds);
|
||||
}
|
||||
{
|
||||
let ds = fb.create_dataset("f64_2d");
|
||||
ds.with_f64_data(&ramp_f64(37 * 53))
|
||||
.with_shape(&[37, 53])
|
||||
.with_chunks(&[10, 16]);
|
||||
configure(ds);
|
||||
}
|
||||
{
|
||||
let ds = fb.create_dataset("u8_1d");
|
||||
ds.with_u8_data(&ramp_u8(5000)).with_chunks(&[777]);
|
||||
configure(ds);
|
||||
}
|
||||
{
|
||||
let ds = fb.create_dataset("f64_big");
|
||||
ds.with_f64_data(&ramp_f64(100_000)).with_chunks(&[40_000]);
|
||||
configure(ds);
|
||||
}
|
||||
fb.write(&path).unwrap();
|
||||
|
||||
// clawhdf5 reads its own output.
|
||||
let file = File::open(&path).unwrap();
|
||||
assert_eq!(
|
||||
file.dataset("i32_1d").unwrap().read_i32().unwrap(),
|
||||
ramp_i32(10_000)
|
||||
);
|
||||
assert_eq!(
|
||||
file.dataset("f64_2d").unwrap().read_f64().unwrap(),
|
||||
ramp_f64(37 * 53)
|
||||
);
|
||||
|
||||
let out = run_python(
|
||||
VERIFY,
|
||||
&[path.to_str().unwrap(), &filter_id.to_string(), filter_name],
|
||||
);
|
||||
assert_eq!(out, "OK", "{tag}: h5py could not read our output");
|
||||
}
|
||||
|
||||
#[cfg(feature = "bitshuffle")]
|
||||
#[test]
|
||||
fn bitshuffle_written_by_hdf5plugin_reads_exactly() {
|
||||
if !have_python("h5py, hdf5plugin") {
|
||||
return;
|
||||
}
|
||||
check_h5py_written(
|
||||
"bitshuffle",
|
||||
r#"[('none', hdf5plugin.Bitshuffle(cname='none')),
|
||||
('lz4', hdf5plugin.Bitshuffle(cname='lz4')),
|
||||
('lz4 nelems=16', hdf5plugin.Bitshuffle(nelems=16, cname='lz4')),
|
||||
('none nelems=64', hdf5plugin.Bitshuffle(nelems=64, cname='none')),
|
||||
('zstd', hdf5plugin.Bitshuffle(cname='zstd')),
|
||||
('zstd clevel=19 nelems=2048', hdf5plugin.Bitshuffle(nelems=2048, cname='zstd', clevel=19))]"#,
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "bitshuffle")]
|
||||
#[test]
|
||||
fn bitshuffle_written_by_clawhdf5_reads_in_hdf5plugin() {
|
||||
use clawhdf5_format::chunked_write::{BitshuffleCompression, PluginFilter};
|
||||
if !have_python("h5py, hdf5plugin") {
|
||||
return;
|
||||
}
|
||||
for (tag, compression) in [
|
||||
("bshuf_none", BitshuffleCompression::None),
|
||||
("bshuf_lz4", BitshuffleCompression::Lz4),
|
||||
("bshuf_zstd", BitshuffleCompression::Zstd { level: 3 }),
|
||||
] {
|
||||
check_ours_read_by_h5py(tag, 32008, "bitshuffle", |ds| {
|
||||
ds.with_bitshuffle(compression);
|
||||
});
|
||||
check_ours_read_by_h5py(tag, 32008, "bitshuffle", |ds| {
|
||||
ds.with_plugin_filter(PluginFilter::Bitshuffle {
|
||||
block_size: 40,
|
||||
compression,
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "bzip2")]
|
||||
#[test]
|
||||
fn bzip2_written_by_hdf5plugin_reads_exactly() {
|
||||
if !have_python("h5py, hdf5plugin") {
|
||||
return;
|
||||
}
|
||||
check_h5py_written(
|
||||
"bzip2",
|
||||
r#"[('bzip2 9', hdf5plugin.BZip2()),
|
||||
('bzip2 1', hdf5plugin.BZip2(blocksize=1)),
|
||||
('bzip2 5 + shuffle', dict(**hdf5plugin.BZip2(blocksize=5), shuffle=True))]"#,
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "bzip2")]
|
||||
#[test]
|
||||
fn bzip2_written_by_clawhdf5_reads_in_hdf5plugin() {
|
||||
if !have_python("h5py, hdf5plugin") {
|
||||
return;
|
||||
}
|
||||
check_ours_read_by_h5py("bzip2", 307, "bzip2", |ds| {
|
||||
ds.with_bzip2(9);
|
||||
});
|
||||
check_ours_read_by_h5py("bzip2_1", 307, "bzip2", |ds| {
|
||||
ds.with_bzip2(1).without_shuffle();
|
||||
});
|
||||
}
|
||||
|
||||
#[cfg(feature = "blosc")]
|
||||
#[test]
|
||||
fn blosc_written_by_hdf5plugin_reads_exactly() {
|
||||
if !have_python("h5py, hdf5plugin") {
|
||||
return;
|
||||
}
|
||||
// Every codec hdf5plugin's Blosc offers, each shuffle mode, and levels
|
||||
// from "store" to maximum.
|
||||
check_h5py_written(
|
||||
"blosc",
|
||||
r#"[(f'{c} {s} {l}', hdf5plugin.Blosc(cname=c, clevel=l, shuffle=s))
|
||||
for c in ['blosclz', 'lz4', 'lz4hc', 'snappy', 'zlib', 'zstd']
|
||||
for s, l in [(hdf5plugin.Blosc.NOSHUFFLE, 5),
|
||||
(hdf5plugin.Blosc.SHUFFLE, 9),
|
||||
(hdf5plugin.Blosc.BITSHUFFLE, 1)]]
|
||||
+ [('blosclz level 0', hdf5plugin.Blosc(cname='blosclz', clevel=0))]"#,
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "blosc")]
|
||||
#[test]
|
||||
fn blosc_written_by_clawhdf5_reads_in_hdf5plugin() {
|
||||
use clawhdf5_format::chunked_write::{BloscCodec, BloscShuffle};
|
||||
if !have_python("h5py, hdf5plugin") {
|
||||
return;
|
||||
}
|
||||
for codec in [
|
||||
BloscCodec::Lz4,
|
||||
BloscCodec::Snappy,
|
||||
BloscCodec::Zlib,
|
||||
BloscCodec::Zstd,
|
||||
] {
|
||||
for (shuffle, level) in [
|
||||
(BloscShuffle::None, 5),
|
||||
(BloscShuffle::Byte, 9),
|
||||
(BloscShuffle::Bit, 1),
|
||||
(BloscShuffle::Byte, 0),
|
||||
] {
|
||||
check_ours_read_by_h5py(
|
||||
&format!("blosc_{codec:?}_{shuffle:?}_{level}"),
|
||||
32001,
|
||||
"blosc",
|
||||
|ds| {
|
||||
ds.with_blosc(codec, level, shuffle);
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "lzf")]
|
||||
#[test]
|
||||
fn lzf_written_by_h5py_reads_exactly() {
|
||||
if !have_python("h5py") {
|
||||
return;
|
||||
}
|
||||
check_h5py_written(
|
||||
"lzf",
|
||||
r#"[('lzf', dict(compression='lzf')),
|
||||
('lzf+shuffle', dict(compression='lzf', shuffle=True)),
|
||||
('lzf+shuffle+fletcher32', dict(compression='lzf', shuffle=True, fletcher32=True))]"#,
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "lzf")]
|
||||
#[test]
|
||||
fn lzf_written_by_clawhdf5_reads_in_h5py() {
|
||||
if !have_python("h5py") {
|
||||
return;
|
||||
}
|
||||
check_ours_read_by_h5py("lzf", 32000, "lzf", |ds| {
|
||||
ds.with_lzf();
|
||||
});
|
||||
check_ours_read_by_h5py("lzf_noshuffle", 32000, "lzf", |ds| {
|
||||
ds.with_lzf().without_shuffle();
|
||||
});
|
||||
}
|
||||
|
||||
/// Blosc2 and ZFP are not implemented: reading them must be a clear error
|
||||
/// naming the filter, never data.
|
||||
#[test]
|
||||
fn unimplemented_filters_are_a_clear_error() {
|
||||
if !have_python("h5py, hdf5plugin") {
|
||||
return;
|
||||
}
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("unimplemented.h5");
|
||||
run_python(
|
||||
r#"
|
||||
import sys
|
||||
import numpy as np, h5py, hdf5plugin
|
||||
with h5py.File(sys.argv[1], 'w') as f:
|
||||
d = np.arange(4096, dtype='<f4').reshape(64, 64)
|
||||
f.create_dataset('blosc2', data=d, chunks=(16, 16), **hdf5plugin.Blosc2())
|
||||
f.create_dataset('zfp', data=d, chunks=(16, 16), **hdf5plugin.Zfp(reversible=True))
|
||||
"#,
|
||||
&[path.to_str().unwrap()],
|
||||
);
|
||||
let file = File::open(&path).unwrap();
|
||||
for (name, id, label) in [("blosc2", 32026u16, "Blosc2"), ("zfp", 32013, "ZFP")] {
|
||||
let err = file
|
||||
.dataset(name)
|
||||
.unwrap()
|
||||
.read_selection(&Selection::All)
|
||||
.expect_err("an unimplemented filter must not read");
|
||||
let msg = err.to_string();
|
||||
assert!(
|
||||
msg.contains(&id.to_string()) && msg.contains(label),
|
||||
"{name}: {msg}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// A corrupt chunk must never read as zeros: a Blosc frame that declares no
|
||||
/// data (libhdf5's filter fails it), and Blosc, LZF and bzip2 chunks that
|
||||
/// decode short (libhdf5 returns the rest of the chunk uninitialised).
|
||||
#[test]
|
||||
fn short_decoding_chunks_are_errors() {
|
||||
if !have_python("h5py, hdf5plugin") {
|
||||
return;
|
||||
}
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("short.h5");
|
||||
run_python(
|
||||
r#"
|
||||
import sys, struct, bz2
|
||||
import numpy as np, h5py, hdf5plugin
|
||||
def lzf(b):
|
||||
# One literal run per 32 bytes: a valid LZF stream.
|
||||
return b"".join(bytes([len(b[i:i+32]) - 1]) + b[i:i+32] for i in range(0, len(b), 32))
|
||||
with h5py.File(sys.argv[1], 'w') as f:
|
||||
kw = dict(shape=(16,), dtype='<i4', chunks=(16,))
|
||||
empty = f.create_dataset('blosc_empty', **kw, **hdf5plugin.Blosc(cname='lz4'))
|
||||
empty.id.write_direct_chunk((0,), bytes([2, 1, 0x20, 4]) + struct.pack('<III', 0, 64, 16))
|
||||
short = f.create_dataset('blosc_short', **kw, **hdf5plugin.Blosc(cname='lz4'))
|
||||
short.id.write_direct_chunk((0,), bytes([2, 1, 0x22, 4]) + struct.pack('<III', 32, 32, 48) + bytes(32))
|
||||
try:
|
||||
f['blosc_empty'][...]
|
||||
except OSError:
|
||||
pass
|
||||
else:
|
||||
raise SystemExit('blosc_empty: libhdf5 read it')
|
||||
lz = f.create_dataset('lzf_short', **kw, compression='lzf')
|
||||
lz.id.write_direct_chunk((0,), lzf(np.arange(8, dtype='<i4').tobytes()))
|
||||
bz = f.create_dataset('bzip2_short', **kw, **hdf5plugin.BZip2())
|
||||
bz.id.write_direct_chunk((0,), bz2.compress(np.arange(8, dtype='<i4').tobytes()))
|
||||
ok = f.create_dataset('lzf_ok', **kw, compression='lzf')
|
||||
ok.id.write_direct_chunk((0,), lzf(np.arange(16, dtype='<i4').tobytes()))
|
||||
"#,
|
||||
&[path.to_str().unwrap()],
|
||||
);
|
||||
let file = File::open(&path).unwrap();
|
||||
for name in ["blosc_empty", "blosc_short", "lzf_short", "bzip2_short"] {
|
||||
let ds = file.dataset(name).unwrap();
|
||||
assert!(
|
||||
ds.read_i32().is_err(),
|
||||
"{name}: {:?}",
|
||||
ds.read_i32().unwrap()
|
||||
);
|
||||
assert!(ds.read_selection(&Selection::All).is_err(), "{name}");
|
||||
}
|
||||
let want: Vec<i32> = (0..16).collect();
|
||||
assert_eq!(file.dataset("lzf_ok").unwrap().read_i32().unwrap(), want);
|
||||
}
|
||||
Reference in New Issue
Block a user