quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests 1 Actions 1 Packages Projects Releases 1 Wiki Activity

feat(format): LZF filter (32000), read and write, pure Rust

Browse Source 
h5py's built-in compression="lzf" failed with UnsupportedFilter(32000). The
new `lzf` feature (no dependencies, on by default in clawhdf5-format and
the facade) decodes the raw liblzf stream h5py's filter stores, bounded by
the chunk size, and encodes it: DatasetBuilder::with_lzf() (or
with_plugin_filter(PluginFilter::Lzf)) writes the filter with h5py's
cd_values (filter version 4, liblzf 0x0105, chunk size in bytes), flagged
optional as h5py does. ChunkOptions gains a `plugin` field for the plugin
filters; build_pipeline_for_chunk passes the chunk size to filters that
record it.

tests/plugin_filters_interop.rs: h5py writes LZF (alone, with shuffle, with
shuffle+fletcher32) over 12 dtype/shape/chunk/data cases with partial edge
chunks and incompressible data, and every dataset reads byte for byte equal
to its unfiltered twin; our LZF output (1-D and 2-D, edge chunks, with and
without shuffle) reads back in h5py. Both fail with the decoder removed.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-26 00:01:22 -05:00
co-authored by Claude Opus 5.5
parent d16544b928
commit e38f9123db
9 changed files with 629 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/chunked_write.rs
+86 -6
View File
@@ -12,7 +12,7 @@ use crate::chunk_grid::ChunkGrid;
use crate::ea_writer;
use crate::error::FormatError;
use crate::filter_pipeline::{
FILTER_DEFLATE, FILTER_FLETCHER32, FILTER_LZ4, FILTER_PCODEC, FILTER_PCODEC_NAME,
FILTER_DEFLATE, FILTER_FLETCHER32, FILTER_LZ4, FILTER_LZF, FILTER_PCODEC, FILTER_PCODEC_NAME,
FILTER_SHUFFLE, FILTER_ZSTD, FilterDescription, FilterPipeline,
};
use crate::filters::compress_chunk;
@@ -48,6 +48,45 @@ pub struct ChunkOptions {
/// Pcodec lossless numerical compression. Private, unregistered filter
/// ID [`FILTER_PCODEC`] (480): only clawhdf5 can read it.
pub pcodec: bool,
/// A plugin compression filter (LZF, ...). Takes priority over the
/// codecs above. Each needs its cargo feature to be written.
pub plugin: Option<PluginFilter>,
}
/// A compression filter from the common HDF5 plugin set, written in the
/// format the libhdf5 plugin (h5py / hdf5plugin) reads.
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub enum PluginFilter {
/// LZF (filter 32000), h5py's built-in `compression="lzf"`. Needs the
/// `lzf` feature.
Lzf,
}
impl PluginFilter {
/// Whether the filter reorders bytes itself, so the automatic shuffle
/// pre-filter would only get in its way.
fn shuffles_itself(&self) -> bool {
match self {
PluginFilter::Lzf => false,
}
}
/// The pipeline entry for this filter. `chunk_bytes` is one chunk's
/// uncompressed size (0 if unknown).
fn description(&self, _element_size: u32, chunk_bytes: u32) -> FilterDescription {
match self {
// h5py's lzf_set_local: filter version, liblzf version, chunk
// size in bytes. Optional, as h5py flags it: a chunk the filter
// cannot shrink may then be stored unfiltered.
PluginFilter::Lzf => FilterDescription {
filter_id: FILTER_LZF,
name: Some("lzf".into()),
flags: 1,
client_data: vec![4, 0x0105, chunk_bytes],
},
}
}
}
/// Largest chunk the automatic choice produces, in bytes.
@@ -92,14 +131,33 @@ impl ChunkOptions {
|| self.lz4
|| self.zstd_level.is_some()
|| self.pcodec
|| self.plugin.is_some()
}
/// Build a FilterPipeline from the options.
pub fn build_pipeline(&self, element_size: u32) -> Option<FilterPipeline> {
self.build_pipeline_for_chunk(element_size, 0)
}
/// Build a FilterPipeline for chunks of `chunk_bytes` uncompressed bytes
/// (0 if unknown). Some plugin filters record the chunk size in their
/// client data.
pub fn build_pipeline_for_chunk(
&self,
element_size: u32,
chunk_bytes: u32,
) -> Option<FilterPipeline> {
let mut filters = Vec::new();
let has_compression =
self.deflate_level.is_some() || self.zstd_level.is_some() || self.lz4 || self.pcodec;
let plugin_shuffles = self
.plugin
.as_ref()
.is_some_and(PluginFilter::shuffles_itself);
let has_compression = self.deflate_level.is_some()
|| self.zstd_level.is_some()
|| self.lz4
|| self.pcodec
|| (self.plugin.is_some() && !plugin_shuffles);
// Shuffle before compression. Applied if explicitly requested OR if compression
// is active and the caller hasn't disabled it — matches h5py default behavior
@@ -113,8 +171,11 @@ impl ChunkOptions {
});
}
// Compression filters (mutually exclusive, priority: pcodec > zstd > lz4 > deflate)
if self.pcodec {
// Compression filters (mutually exclusive, priority: plugin > pcodec >
// zstd > lz4 > deflate)
if let Some(plugin) = &self.plugin {
filters.push(plugin.description(element_size, chunk_bytes));
} else if self.pcodec {
filters.push(FilterDescription {
filter_id: FILTER_PCODEC,
name: Some(FILTER_PCODEC_NAME.into()),
@@ -675,7 +736,12 @@ pub fn precompress_chunks(
element_size: usize,
options: &ChunkOptions,
) -> Result<PrecompressedChunks, FormatError> {
let pipeline = options.build_pipeline(element_size as u32);
let chunk_bytes = chunk_dims
.iter()
.try_fold(element_size as u64, |acc, &d| acc.checked_mul(d))
.and_then(|b| u32::try_from(b).ok())
.unwrap_or(0);
let pipeline = options.build_pipeline_for_chunk(element_size as u32, chunk_bytes);
let has_filters = pipeline.is_some();
let pipeline_message = pipeline.as_ref().map(|pl| pl.serialize());
@@ -1569,6 +1635,20 @@ mod tests {
assert_eq!(pl.filters[1].client_data, vec![3]);
}
#[test]
fn chunk_options_pipeline_lzf() {
let options = ChunkOptions {
plugin: Some(PluginFilter::Lzf),
..Default::default()
};
assert!(options.is_chunked());
let pl = options.build_pipeline_for_chunk(8, 800).unwrap();
assert_eq!(pl.filters.len(), 2);
assert_eq!(pl.filters[0].filter_id, FILTER_SHUFFLE);
assert_eq!(pl.filters[1].filter_id, FILTER_LZF);
assert_eq!(pl.filters[1].client_data, vec![4, 0x0105, 800]);
}
#[test]
fn chunk_options_zstd_priority_over_deflate() {
let options = ChunkOptions {
crates/clawhdf5-format/src/filter_pipeline.rs
+12
View File
@@ -19,6 +19,18 @@ pub const FILTER_SCALEOFFSET: u16 = 6;
pub const FILTER_LZ4: u16 = 32004;
/// Zstandard compression.
pub const FILTER_ZSTD: u16 = 32015;
/// bzip2 (registered by PyTables; hdf5plugin's `BZip2`).
pub const FILTER_BZIP2: u16 = 307;
/// LZF — h5py's built-in `compression="lzf"`.
pub const FILTER_LZF: u16 = 32000;
/// Blosc 1 (hdf5-blosc; hdf5plugin's `Blosc`).
pub const FILTER_BLOSC: u16 = 32001;
/// Bitshuffle, optionally with LZ4 or Zstandard (hdf5plugin's `Bitshuffle`).
pub const FILTER_BITSHUFFLE: u16 = 32008;
/// ZFP lossy floating-point compression (hdf5plugin's `Zfp`). Not supported.
pub const FILTER_ZFP: u16 = 32013;
/// Blosc 2 (hdf5plugin's `Blosc2`).
pub const FILTER_BLOSC2: u16 = 32026;
/// Pcodec lossless numerical codec — a **private, unregistered** clawhdf5
/// filter. Pcodec has no ID in the HDF Group's filter registry (checked
/// 2026-09-25, `hdf5_plugins/docs/RegisteredFilterPlugins.md`), so it uses an
crates/clawhdf5-format/src/filters.rs
+8 -1
View File
@@ -192,6 +192,13 @@ pub(crate) static BUILTIN_FILTERS: &[BuiltinFilter] = &[
decode: |d, c| pcodec_decompress(d, c.element_size, c.max_output),
encode: Some(|d, c| pcodec_compress(d, c.element_size)),
},
#[cfg(feature = "lzf")]
BuiltinFilter {
id: crate::filter_pipeline::FILTER_LZF,
name: "lzf",
decode: crate::filters_lzf::lzf_decode,
encode: Some(crate::filters_lzf::lzf_encode),
},
#[cfg(feature = "lz4")]
BuiltinFilter {
id: FILTER_LZ4,
@@ -1702,7 +1709,7 @@ mod tests {
// An unsupported filter is fine when the chunk skipped it.
let unknown = FilterPipeline {
version: 2,
filters: vec![filter(32000), filter(FILTER_DEFLATE)],
filters: vec![filter(32013), filter(FILTER_DEFLATE)],
};
assert_eq!(
decompress_chunk_masked(&deflated, &unknown, n, 8, 0b01).unwrap(),
crates/clawhdf5-format/src/filters_lzf.rs
+232
View File
@@ -0,0 +1,232 @@
//! LZF (HDF5 filter 32000) — h5py's built-in compression filter
//! (`compression="lzf"`), in pure Rust.
//!
//! The chunk is one raw LZF stream (liblzf 3.x format, no header). The
//! stream is a sequence of instructions, each starting with a control byte:
//!
//! * `000LLLLL` — a literal run: the next `L + 1` bytes (1..=32) are copied.
//! * `LLLOOOOO [E] OOOOOOOO` — a back reference: copy `len + 2` bytes from
//! `distance` bytes back, where `len` is the top three bits (1..=6), or
//! `7 + E` when they are all ones, and `distance` is the 13-bit offset
//! (high five bits in the control byte, low eight in the last byte) plus 1.
//!
//! h5py's filter (`lzf_filter.c`) records the chunk's size in bytes in
//! `cd_values[2]` (slots 0 and 1 hold the filter and liblzf versions) and
//! sizes its output buffer from it.
#[cfg(not(feature = "std"))]
extern crate alloc;
#[cfg(not(feature = "std"))]
use alloc::{format, vec, vec::Vec};
use crate::error::FormatError;
use crate::filter_registry::FilterContext;
/// `H5PY_FILTER_LZF_VERSION`, written to `cd_values[0]`.
pub const LZF_FILTER_VERSION: u32 = 4;
/// `LZF_VERSION` (liblzf 1.5), written to `cd_values[1]`.
pub const LZF_API_VERSION: u32 = 0x0105;
const MAX_LITERAL: usize = 32;
const MAX_OFFSET: usize = 1 << 13;
const MAX_REF: usize = (1 << 8) + (1 << 3);
const HASH_LOG: u32 = 14;
fn err(msg: &str) -> FormatError {
FormatError::DecompressionError(format!("lzf: {msg}"))
}
/// Decode an LZF-filtered chunk.
pub(crate) fn lzf_decode(input: &[u8], ctx: &FilterContext<'_>) -> Result<Vec<u8>, FormatError> {
let limit = ctx.output_limit();
let hint = match ctx.client_data().get(2) {
Some(&n) if n != 0 => n as usize,
_ => input.len().saturating_mul(2),
};
lzf_decompress(input, hint.min(limit), limit)
}
/// Decompress a raw LZF stream, refusing to produce more than `limit` bytes.
pub fn lzf_decompress(
input: &[u8],
size_hint: usize,
limit: usize,
) -> Result<Vec<u8>, FormatError> {
let mut out: Vec<u8> = Vec::new();
out.try_reserve(size_hint)
.map_err(|_| err("cannot allocate the output buffer"))?;
let mut ip = 0usize;
while ip < input.len() {
let ctrl = input[ip] as usize;
ip += 1;
if ctrl < 32 {
let run = ctrl + 1;
let lit = input
.get(ip..ip + run)
.ok_or_else(|| err("literal run past the end of the input"))?;
if out.len() + run > limit {
return Err(err("output exceeds the chunk size"));
}
out.extend_from_slice(lit);
ip += run;
} else {
let mut len = ctrl >> 5;
if len == 7 {
len += *input
.get(ip)
.ok_or_else(|| err("truncated back reference"))?
as usize;
ip += 1;
}
let low = *input
.get(ip)
.ok_or_else(|| err("truncated back reference"))? as usize;
ip += 1;
let distance = ((ctrl & 0x1f) << 8) + low + 1;
let len = len + 2;
if distance > out.len() {
return Err(err("back reference before the start of the output"));
}
if out.len() + len > limit {
return Err(err("output exceeds the chunk size"));
}
let start = out.len() - distance;
if distance >= len {
out.extend_from_within(start..start + len);
} else {
// Overlapping copy: repeats the last `distance` bytes.
for k in 0..len {
let b = out[start + k];
out.push(b);
}
}
}
}
Ok(out)
}
/// Encode a chunk with the LZF filter.
pub(crate) fn lzf_encode(input: &[u8], _ctx: &FilterContext<'_>) -> Result<Vec<u8>, FormatError> {
Ok(lzf_compress(input))
}
fn hash3(b: &[u8]) -> usize {
let v = (u32::from(b[0]) << 16) | (u32::from(b[1]) << 8) | u32::from(b[2]);
(v.wrapping_mul(2_654_435_761) >> (32 - HASH_LOG)) as usize
}
fn flush_literals(out: &mut Vec<u8>, lit: &[u8]) {
for run in lit.chunks(MAX_LITERAL) {
out.push((run.len() - 1) as u8);
out.extend_from_slice(run);
}
}
/// Compress `input` into a raw LZF stream any liblzf decoder reads.
///
/// Incompressible input grows by one byte per 32. (h5py's own filter gives
/// up on such a chunk and stores it unfiltered; storing the slightly larger
/// stream is equally readable.)
pub fn lzf_compress(input: &[u8]) -> Vec<u8> {
let n = input.len();
let mut out = Vec::with_capacity(n + n / MAX_LITERAL + 1);
let mut table = vec![0u32; 1 << HASH_LOG];
let mut lit_start = 0usize;
let mut i = 0usize;
while i + 2 < n {
let h = hash3(&input[i..]);
let cand = table[h] as usize;
table[h] = (i + 1) as u32;
if cand != 0 {
let r = cand - 1;
let distance = i - r;
if distance <= MAX_OFFSET && input[r..r + 3] == input[i..i + 3] {
let max_len = (n - i).min(MAX_REF);
let mut len = 3;
while len < max_len && input[r + len] == input[i + len] {
len += 1;
}
flush_literals(&mut out, &input[lit_start..i]);
let code = len - 2;
let off = distance - 1;
if code < 7 {
out.push(((code << 5) | (off >> 8)) as u8);
} else {
out.push(((7 << 5) | (off >> 8)) as u8);
out.push((code - 7) as u8);
}
out.push((off & 0xff) as u8);
// Index the positions the match covered so later data can
// refer back into it.
let end = i + len;
let mut j = i + 1;
while j < end && j + 2 < n {
table[hash3(&input[j..])] = (j + 1) as u32;
j += 1;
}
i = end;
lit_start = i;
continue;
}
}
i += 1;
}
flush_literals(&mut out, &input[lit_start..]);
out
}
#[cfg(test)]
mod tests {
use super::*;
fn round_trip(data: &[u8]) {
let c = lzf_compress(data);
assert_eq!(lzf_decompress(&c, data.len(), data.len()).unwrap(), data);
}
#[test]
fn round_trips() {
round_trip(b"");
round_trip(b"a");
round_trip(b"abcabcabcabcabcabcabcabcabcabcabcabc");
round_trip(&[7u8; 10_000]);
let noise: Vec<u8> = (0..70_000u32)
.map(|i| (i.wrapping_mul(2_654_435_761) >> 13) as u8)
.collect();
round_trip(&noise);
let ramp: Vec<u8> = (0..100_000u32)
.flat_map(|i| (i % 1000).to_le_bytes())
.collect();
round_trip(&ramp);
}
#[test]
fn compresses_repetitive_data() {
let data = [42u8; 4096];
assert!(lzf_compress(&data).len() < 100);
}
/// The chunk h5py 3.16's bundled liblzf writes for
/// `b"hello hello hello hello"` (read back with `read_direct_chunk`): a
/// 7-byte literal, a 14-byte back reference 6 bytes back (extended
/// length), and a 2-byte literal.
#[test]
fn decodes_liblzf_output() {
let stream = b"\x06hello h\xe0\x05\x05\x01lo";
assert_eq!(
lzf_decompress(stream, 23, 23).unwrap(),
b"hello hello hello hello"
);
}
#[test]
fn rejects_corrupt_streams() {
// Back reference before the start.
assert!(lzf_decompress(&[0x20, 0x00], 10, 10).is_err());
// Literal run past the end.
assert!(lzf_decompress(&[0x05, 1, 2], 10, 10).is_err());
// Output over the limit.
let c = lzf_compress(&[1u8; 100]);
assert!(lzf_decompress(&c, 10, 99).is_err());
}
}
crates/clawhdf5-format/src/lib.rs
+2
View File
@@ -73,6 +73,8 @@ pub mod fill_value;
pub mod filter_pipeline;
pub mod filter_registry;
pub mod filters;
#[cfg(feature = "lzf")]
pub mod filters_lzf;
mod filters_szip;
pub mod fixed_array;
pub mod float16;
crates/clawhdf5-format/src/type_builders.rs
+18
View File
@@ -731,6 +731,24 @@ impl DatasetBuilder {
self
}
/// Compress with a plugin filter ([`PluginFilter`]), in the format the
/// libhdf5 plugin reads (h5py, hdf5plugin). Implies chunked storage.
/// Each filter needs its cargo feature (`lzf`, ...); writing fails with
/// `UnsupportedFilter` without it.
///
/// [`PluginFilter`]: crate::chunked_write::PluginFilter
pub fn with_plugin_filter(&mut self, filter: crate::chunked_write::PluginFilter) -> &mut Self {
self.chunk_options.plugin = Some(filter);
self
}
/// Enable LZF compression (filter 32000) — h5py's built-in
/// `compression="lzf"`. Implies chunked storage; shuffle is applied
/// first unless `.without_shuffle()`. Requires the `lzf` cargo feature.
pub fn with_lzf(&mut self) -> &mut Self {
self.with_plugin_filter(crate::chunked_write::PluginFilter::Lzf)
}
/// Enable Pcodec lossless numerical compression (private clawhdf5 filter
/// ID 480).
///