quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Merge branch 'feat/p1-plugin-filters' into feat/p1-proof

Browse Source 
# Conflicts:
#	crates/clawhdf5/tests/h5py_chunked_read_tests.rs
#	docs/known-issues.md
This commit is contained in:
osobh
2026-09-26 01:37:58 -05:00
parent b9a2ce3077 17fc8b1964
commit 591aa71d12
27 changed files with 3492 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/chunked_write.rs
+225 -7
View File
@@ -12,8 +12,9 @@ use crate::chunk_grid::ChunkGrid;
use crate::ea_writer;
use crate::error::FormatError;
use crate::filter_pipeline::{
FILTER_DEFLATE, FILTER_FLETCHER32, FILTER_LZ4, FILTER_PCODEC, FILTER_PCODEC_NAME,
FILTER_SHUFFLE, FILTER_ZSTD, FilterDescription, FilterPipeline,
FILTER_BITSHUFFLE, FILTER_BLOSC, FILTER_BZIP2, FILTER_DEFLATE, FILTER_FLETCHER32, FILTER_LZ4,
FILTER_LZF, FILTER_PCODEC, FILTER_PCODEC_NAME, FILTER_SHUFFLE, FILTER_ZSTD, FilterDescription,
FilterPipeline,
};
use crate::filters::compress_chunk;
/// Round a file offset up to the next cache-line boundary.
@@ -48,6 +49,167 @@ pub struct ChunkOptions {
/// Pcodec lossless numerical compression. Private, unregistered filter
/// ID [`FILTER_PCODEC`] (480): only clawhdf5 can read it.
pub pcodec: bool,
/// A plugin compression filter (LZF, ...). Takes priority over the
/// codecs above. Each needs its cargo feature to be written.
pub plugin: Option<PluginFilter>,
}
/// A compression filter from the common HDF5 plugin set, written in the
/// format the libhdf5 plugin (h5py / hdf5plugin) reads.
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub enum PluginFilter {
/// LZF (filter 32000), h5py's built-in `compression="lzf"`. Needs the
/// `lzf` feature.
Lzf,
/// Bitshuffle (filter 32008): a bit transpose of each block of
/// `block_size` elements (0 = bitshuffle's default, else a multiple of
/// 8), optionally compressed. Needs the `bitshuffle` feature.
Bitshuffle {
/// Block size in elements; 0 for the default.
block_size: u32,
/// Compression after the transpose.
compression: BitshuffleCompression,
},
/// bzip2 (filter 307) at block size `level` (1-9). Needs the `bzip2`
/// feature.
Bzip2 {
/// Block size 1-9 (9 = hdf5plugin's default).
level: u32,
},
/// Blosc 1 (filter 32001): `codec` at `level` (0-9; 0 stores), after
/// `shuffle`. Needs the `blosc` feature.
Blosc {
/// The codec inside the Blosc frame.
codec: BloscCodec,
/// Compression level 0-9 (0 stores the data uncompressed).
level: u32,
/// The shuffle Blosc applies first.
shuffle: BloscShuffle,
},
}
/// The codec inside a Blosc frame that clawhdf5 can write. (It reads
/// BloscLZ too, but cannot write it.)
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum BloscCodec {
/// LZ4.
Lz4,
/// Snappy.
Snappy,
/// Zlib, at the Blosc level.
Zlib,
/// Zstandard (clawhdf5's pure-Rust encoder has one level, about zstd 1).
Zstd,
}
/// The shuffle Blosc applies before compressing.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum BloscShuffle {
/// None.
None,
/// Byte shuffle (Blosc's default).
Byte,
/// Bit shuffle.
Bit,
}
/// What bitshuffle compresses its blocks with.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum BitshuffleCompression {
/// Transpose only.
None,
/// LZ4 (bitshuffle's `cname="lz4"`, the common choice).
Lz4,
/// Zstandard. clawhdf5's pure-Rust encoder has a single level (about
/// zstd's level 1); `level` is recorded in the file for other writers.
Zstd {
/// Level recorded in `cd_values[5]`.
level: u32,
},
}
impl PluginFilter {
/// Whether the filter reorders bytes itself, so the automatic shuffle
/// pre-filter would only get in its way.
fn shuffles_itself(&self) -> bool {
match self {
PluginFilter::Lzf => false,
PluginFilter::Bitshuffle { .. } => true,
PluginFilter::Bzip2 { .. } => false,
PluginFilter::Blosc { .. } => true,
}
}
/// The pipeline entry for this filter. `chunk_bytes` is one chunk's
/// uncompressed size (0 if unknown).
fn description(&self, element_size: u32, chunk_bytes: u32) -> FilterDescription {
match self {
// h5py's lzf_set_local: filter version, liblzf version, chunk
// size in bytes. Optional, as h5py flags it: a chunk the filter
// cannot shrink may then be stored unfiltered.
PluginFilter::Lzf => FilterDescription {
filter_id: FILTER_LZF,
name: Some("lzf".into()),
flags: 1,
client_data: vec![4, 0x0105, chunk_bytes],
},
// bshuf_h5_set_local: version 0.4, element size, block size,
// compression (0 none, 2 LZ4, 3 Zstandard), Zstandard level.
// hdf5-blosc's blosc_set_local: filter revision 2, Blosc format
// 2, type size, chunk size, then level, shuffle, compressor.
PluginFilter::Blosc {
codec,
level,
shuffle,
} => FilterDescription {
filter_id: FILTER_BLOSC,
name: Some("blosc".into()),
flags: 1,
client_data: vec![
2,
2,
element_size,
chunk_bytes,
(*level).min(9),
match shuffle {
BloscShuffle::None => 0,
BloscShuffle::Byte => 1,
BloscShuffle::Bit => 2,
},
match codec {
BloscCodec::Lz4 => 1,
BloscCodec::Snappy => 3,
BloscCodec::Zlib => 4,
BloscCodec::Zstd => 5,
},
],
},
PluginFilter::Bzip2 { level } => FilterDescription {
filter_id: FILTER_BZIP2,
name: Some("bzip2".into()),
flags: 1,
client_data: vec![(*level).clamp(1, 9)],
},
PluginFilter::Bitshuffle {
block_size,
compression,
} => {
let mut cd = vec![0, 4, element_size, *block_size];
match compression {
BitshuffleCompression::None => cd.push(0),
BitshuffleCompression::Lz4 => cd.push(2),
BitshuffleCompression::Zstd { level } => cd.extend([3, *level]),
}
FilterDescription {
filter_id: FILTER_BITSHUFFLE,
name: Some("bitshuffle; see https://github.com/kiyo-masui/bitshuffle".into()),
flags: 1,
client_data: cd,
}
}
}
}
}
/// Largest chunk the automatic choice produces, in bytes.
@@ -92,14 +254,33 @@ impl ChunkOptions {
|| self.lz4
|| self.zstd_level.is_some()
|| self.pcodec
|| self.plugin.is_some()
}
/// Build a FilterPipeline from the options.
pub fn build_pipeline(&self, element_size: u32) -> Option<FilterPipeline> {
self.build_pipeline_for_chunk(element_size, 0)
}
/// Build a FilterPipeline for chunks of `chunk_bytes` uncompressed bytes
/// (0 if unknown). Some plugin filters record the chunk size in their
/// client data.
pub fn build_pipeline_for_chunk(
&self,
element_size: u32,
chunk_bytes: u32,
) -> Option<FilterPipeline> {
let mut filters = Vec::new();
let has_compression =
self.deflate_level.is_some() || self.zstd_level.is_some() || self.lz4 || self.pcodec;
let plugin_shuffles = self
.plugin
.as_ref()
.is_some_and(PluginFilter::shuffles_itself);
let has_compression = self.deflate_level.is_some()
|| self.zstd_level.is_some()
|| self.lz4
|| self.pcodec
|| (self.plugin.is_some() && !plugin_shuffles);
// Shuffle before compression. Applied if explicitly requested OR if compression
// is active and the caller hasn't disabled it — matches h5py default behavior
@@ -113,8 +294,11 @@ impl ChunkOptions {
});
}
// Compression filters (mutually exclusive, priority: pcodec > zstd > lz4 > deflate)
if self.pcodec {
// Compression filters (mutually exclusive, priority: plugin > pcodec >
// zstd > lz4 > deflate)
if let Some(plugin) = &self.plugin {
filters.push(plugin.description(element_size, chunk_bytes));
} else if self.pcodec {
filters.push(FilterDescription {
filter_id: FILTER_PCODEC,
name: Some(FILTER_PCODEC_NAME.into()),
@@ -634,7 +818,12 @@ pub fn precompress_chunks(
element_size: usize,
options: &ChunkOptions,
) -> Result<PrecompressedChunks, FormatError> {
let pipeline = options.build_pipeline(element_size as u32);
let chunk_bytes = chunk_dims
.iter()
.try_fold(element_size as u64, |acc, &d| acc.checked_mul(d))
.and_then(|b| u32::try_from(b).ok())
.unwrap_or(0);
let pipeline = options.build_pipeline_for_chunk(element_size as u32, chunk_bytes);
let has_filters = pipeline.is_some();
let pipeline_message = pipeline.as_ref().map(|pl| pl.serialize());
@@ -1528,6 +1717,35 @@ mod tests {
assert_eq!(pl.filters[1].client_data, vec![3]);
}
#[test]
fn chunk_options_pipeline_lzf() {
let options = ChunkOptions {
plugin: Some(PluginFilter::Lzf),
..Default::default()
};
assert!(options.is_chunked());
let pl = options.build_pipeline_for_chunk(8, 800).unwrap();
assert_eq!(pl.filters.len(), 2);
assert_eq!(pl.filters[0].filter_id, FILTER_SHUFFLE);
assert_eq!(pl.filters[1].filter_id, FILTER_LZF);
assert_eq!(pl.filters[1].client_data, vec![4, 0x0105, 800]);
}
#[test]
fn chunk_options_pipeline_bitshuffle_has_no_auto_shuffle() {
let options = ChunkOptions {
plugin: Some(PluginFilter::Bitshuffle {
block_size: 0,
compression: BitshuffleCompression::Zstd { level: 5 },
}),
..Default::default()
};
let pl = options.build_pipeline(4).unwrap();
assert_eq!(pl.filters.len(), 1);
assert_eq!(pl.filters[0].filter_id, FILTER_BITSHUFFLE);
assert_eq!(pl.filters[0].client_data, vec![0, 4, 4, 0, 3, 5]);
}
#[test]
fn chunk_options_zstd_priority_over_deflate() {
let options = ChunkOptions {