feat(format): bitshuffle filter (32008), with its LZ4 and Zstandard modes
hdf5plugin's Bitshuffle failed with UnsupportedFilter(32008). The new
`bitshuffle` feature (pure Rust: lz4_flex, and ruzstd for Zstandard — the
`zstd` feature's libzstd is not needed) decodes all three modes of
bshuf_h5filter.c — transpose only, LZ4 and Zstandard blocks behind the
12-byte header — including the default and explicit block sizes, the
shorter last block rounded down to a multiple of 8 elements, and the
untransposed trailing elements. Sizes read from the chunk are bounded by
the chunk size.
It also encodes: DatasetBuilder::with_bitshuffle(BitshuffleCompression)
or PluginFilter::Bitshuffle { block_size, compression } writes the filter
with hdf5plugin's cd_values and no automatic byte shuffle. ruzstd has one
compression level (about zstd's 1); the requested level is recorded.
The bit transpose is checked bit for bit against a one-bit-at-a-time model
(which matched hdf5plugin's output) and is shared with blosc next.
Interop: hdf5plugin writes none/LZ4/Zstandard at default and explicit
block sizes and levels over the 12-case matrix, read byte for byte; our
three modes at two block sizes read back through hdf5plugin. Both fail
with the decoder removed.
Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
@@ -22,6 +22,9 @@ zstd = { version = "0.13", optional = true }
|
||||
blake3 = { version = "1", optional = true }
|
||||
libaec-sys = { path = "../libaec-sys", version = "0.1", optional = true }
|
||||
pco = { version = "1.0", optional = true }
|
||||
# Pure-Rust Zstandard, for the plugin filters that embed zstd (bitshuffle,
|
||||
# blosc). The `zstd` feature (filter 32015) links libzstd instead.
|
||||
ruzstd = { version = "0.9", optional = true }
|
||||
|
||||
[dev-dependencies]
|
||||
half = { workspace = true }
|
||||
@@ -59,6 +62,8 @@ pcodec = ["dep:pco"]
|
||||
# Plugin filters, pure Rust. LZF (32000) is h5py's built-in compression; it
|
||||
# has no dependencies, so it is on by default.
|
||||
lzf = []
|
||||
# Bitshuffle (32008), with its LZ4 and Zstandard modes.
|
||||
bitshuffle = ["lz4_flex", "ruzstd"]
|
||||
|
||||
[[bench]]
|
||||
name = "parallel_decompress_bench"
|
||||
|
||||
@@ -12,8 +12,8 @@ use crate::chunk_grid::ChunkGrid;
|
||||
use crate::ea_writer;
|
||||
use crate::error::FormatError;
|
||||
use crate::filter_pipeline::{
|
||||
FILTER_DEFLATE, FILTER_FLETCHER32, FILTER_LZ4, FILTER_LZF, FILTER_PCODEC, FILTER_PCODEC_NAME,
|
||||
FILTER_SHUFFLE, FILTER_ZSTD, FilterDescription, FilterPipeline,
|
||||
FILTER_BITSHUFFLE, FILTER_DEFLATE, FILTER_FLETCHER32, FILTER_LZ4, FILTER_LZF, FILTER_PCODEC,
|
||||
FILTER_PCODEC_NAME, FILTER_SHUFFLE, FILTER_ZSTD, FilterDescription, FilterPipeline,
|
||||
};
|
||||
use crate::filters::compress_chunk;
|
||||
/// Round a file offset up to the next cache-line boundary.
|
||||
@@ -61,6 +61,30 @@ pub enum PluginFilter {
|
||||
/// LZF (filter 32000), h5py's built-in `compression="lzf"`. Needs the
|
||||
/// `lzf` feature.
|
||||
Lzf,
|
||||
/// Bitshuffle (filter 32008): a bit transpose of each block of
|
||||
/// `block_size` elements (0 = bitshuffle's default, else a multiple of
|
||||
/// 8), optionally compressed. Needs the `bitshuffle` feature.
|
||||
Bitshuffle {
|
||||
/// Block size in elements; 0 for the default.
|
||||
block_size: u32,
|
||||
/// Compression after the transpose.
|
||||
compression: BitshuffleCompression,
|
||||
},
|
||||
}
|
||||
|
||||
/// What bitshuffle compresses its blocks with.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum BitshuffleCompression {
|
||||
/// Transpose only.
|
||||
None,
|
||||
/// LZ4 (bitshuffle's `cname="lz4"`, the common choice).
|
||||
Lz4,
|
||||
/// Zstandard. clawhdf5's pure-Rust encoder has a single level (about
|
||||
/// zstd's level 1); `level` is recorded in the file for other writers.
|
||||
Zstd {
|
||||
/// Level recorded in `cd_values[5]`.
|
||||
level: u32,
|
||||
},
|
||||
}
|
||||
|
||||
impl PluginFilter {
|
||||
@@ -69,12 +93,13 @@ impl PluginFilter {
|
||||
fn shuffles_itself(&self) -> bool {
|
||||
match self {
|
||||
PluginFilter::Lzf => false,
|
||||
PluginFilter::Bitshuffle { .. } => true,
|
||||
}
|
||||
}
|
||||
|
||||
/// The pipeline entry for this filter. `chunk_bytes` is one chunk's
|
||||
/// uncompressed size (0 if unknown).
|
||||
fn description(&self, _element_size: u32, chunk_bytes: u32) -> FilterDescription {
|
||||
fn description(&self, element_size: u32, chunk_bytes: u32) -> FilterDescription {
|
||||
match self {
|
||||
// h5py's lzf_set_local: filter version, liblzf version, chunk
|
||||
// size in bytes. Optional, as h5py flags it: a chunk the filter
|
||||
@@ -85,6 +110,25 @@ impl PluginFilter {
|
||||
flags: 1,
|
||||
client_data: vec![4, 0x0105, chunk_bytes],
|
||||
},
|
||||
// bshuf_h5_set_local: version 0.4, element size, block size,
|
||||
// compression (0 none, 2 LZ4, 3 Zstandard), Zstandard level.
|
||||
PluginFilter::Bitshuffle {
|
||||
block_size,
|
||||
compression,
|
||||
} => {
|
||||
let mut cd = vec![0, 4, element_size, *block_size];
|
||||
match compression {
|
||||
BitshuffleCompression::None => cd.push(0),
|
||||
BitshuffleCompression::Lz4 => cd.push(2),
|
||||
BitshuffleCompression::Zstd { level } => cd.extend([3, *level]),
|
||||
}
|
||||
FilterDescription {
|
||||
filter_id: FILTER_BITSHUFFLE,
|
||||
name: Some("bitshuffle; see https://github.com/kiyo-masui/bitshuffle".into()),
|
||||
flags: 1,
|
||||
client_data: cd,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1649,6 +1693,21 @@ mod tests {
|
||||
assert_eq!(pl.filters[1].client_data, vec![4, 0x0105, 800]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chunk_options_pipeline_bitshuffle_has_no_auto_shuffle() {
|
||||
let options = ChunkOptions {
|
||||
plugin: Some(PluginFilter::Bitshuffle {
|
||||
block_size: 0,
|
||||
compression: BitshuffleCompression::Zstd { level: 5 },
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
let pl = options.build_pipeline(4).unwrap();
|
||||
assert_eq!(pl.filters.len(), 1);
|
||||
assert_eq!(pl.filters[0].filter_id, FILTER_BITSHUFFLE);
|
||||
assert_eq!(pl.filters[0].client_data, vec![0, 4, 4, 0, 3, 5]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chunk_options_zstd_priority_over_deflate() {
|
||||
let options = ChunkOptions {
|
||||
|
||||
@@ -206,6 +206,13 @@ pub(crate) static BUILTIN_FILTERS: &[BuiltinFilter] = &[
|
||||
decode: |d, c| lz4_decompress(d, c.max_output),
|
||||
encode: Some(|d, c| lz4_compress(d, c.client_data())),
|
||||
},
|
||||
#[cfg(feature = "bitshuffle")]
|
||||
BuiltinFilter {
|
||||
id: crate::filter_pipeline::FILTER_BITSHUFFLE,
|
||||
name: "bitshuffle",
|
||||
decode: crate::filters_bitshuffle::bitshuffle_decode,
|
||||
encode: Some(crate::filters_bitshuffle::bitshuffle_encode),
|
||||
},
|
||||
#[cfg(feature = "zstd")]
|
||||
BuiltinFilter {
|
||||
id: FILTER_ZSTD,
|
||||
|
||||
@@ -0,0 +1,377 @@
|
||||
//! Bitshuffle (HDF5 filter 32008) and the bit transpose it shares with blosc.
|
||||
//!
|
||||
//! **The transform.** A block of `n` elements (`n` a multiple of 8) of
|
||||
//! `es` bytes each is viewed as an `n × 8·es` bit matrix — row *i* is
|
||||
//! element *i*, column `8·j + k` is bit *k* (LSB first) of its byte *j* — and
|
||||
//! transposed: the output is `8·es` rows of `n` bits, row `8·j + k` holding
|
||||
//! bit *k* of byte *j* of every element in order, packed LSB first. That is
|
||||
//! what `bshuf_trans_bit_elem` produces (checked against hdf5plugin's
|
||||
//! library bit for bit).
|
||||
//!
|
||||
//! **The filter** (`bshuf_h5filter.c`). `cd_values`: `[0..2]` bitshuffle
|
||||
//! version, `[2]` element size, `[3]` block size in elements (0 = default:
|
||||
//! 8192 bytes' worth, rounded down to a multiple of 8, at least 128),
|
||||
//! `[4]` compression (0 none, 2 LZ4, 3 Zstandard), `[5]` Zstandard level.
|
||||
//! The chunk is cut into blocks of `block size` elements; the tail shorter
|
||||
//! than a block is transposed as one block rounded down to a multiple of 8
|
||||
//! elements, and the last `n mod 8` elements are stored as they are.
|
||||
//! Uncompressed, that is the whole chunk. Compressed, the chunk starts with a
|
||||
//! 12-byte header — the decoded size (u64 big-endian) and the block size in
|
||||
//! bytes (u32 big-endian) — and each transposed block is stored as a u32
|
||||
//! big-endian length and an LZ4 block / Zstandard frame; the untransposed
|
||||
//! tail follows the last block.
|
||||
|
||||
#[cfg(not(feature = "std"))]
|
||||
extern crate alloc;
|
||||
#[cfg(not(feature = "std"))]
|
||||
use alloc::{format, vec, vec::Vec};
|
||||
|
||||
use crate::error::FormatError;
|
||||
#[cfg(feature = "bitshuffle")]
|
||||
use crate::filter_registry::FilterContext;
|
||||
|
||||
/// Transpose an 8×8 bit matrix packed in a u64 (byte *r* = row *r*, bit *c*
|
||||
/// of that byte = column *c*). An involution.
|
||||
#[inline]
|
||||
fn transpose8(mut x: u64) -> u64 {
|
||||
let t = (x ^ (x >> 7)) & 0x00AA_00AA_00AA_00AA;
|
||||
x = x ^ t ^ (t << 7);
|
||||
let t = (x ^ (x >> 14)) & 0x0000_CCCC_0000_CCCC;
|
||||
x = x ^ t ^ (t << 14);
|
||||
let t = (x ^ (x >> 28)) & 0x0000_0000_F0F0_F0F0;
|
||||
x ^ t ^ (t << 28)
|
||||
}
|
||||
|
||||
/// Bit-transpose one block: `input` and `out` are `n * es` bytes, `n` a
|
||||
/// multiple of 8.
|
||||
pub(crate) fn bitshuffle_block(input: &[u8], out: &mut [u8], n: usize, es: usize) {
|
||||
debug_assert!(n.is_multiple_of(8) && input.len() == n * es && out.len() == n * es);
|
||||
let row = n / 8;
|
||||
for j in 0..es {
|
||||
for g in 0..row {
|
||||
let mut x = 0u64;
|
||||
for t in 0..8 {
|
||||
x |= u64::from(input[(8 * g + t) * es + j]) << (8 * t);
|
||||
}
|
||||
let y = transpose8(x);
|
||||
for k in 0..8 {
|
||||
out[(8 * j + k) * row + g] = (y >> (8 * k)) as u8;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Undo [`bitshuffle_block`].
|
||||
pub(crate) fn bitunshuffle_block(input: &[u8], out: &mut [u8], n: usize, es: usize) {
|
||||
debug_assert!(n.is_multiple_of(8) && input.len() == n * es && out.len() == n * es);
|
||||
let row = n / 8;
|
||||
for j in 0..es {
|
||||
for g in 0..row {
|
||||
let mut y = 0u64;
|
||||
for k in 0..8 {
|
||||
y |= u64::from(input[(8 * j + k) * row + g]) << (8 * k);
|
||||
}
|
||||
let x = transpose8(y);
|
||||
for t in 0..8 {
|
||||
out[(8 * g + t) * es + j] = (x >> (8 * t)) as u8;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `bshuf_default_block_size`: 8 KiB of elements, a multiple of 8, >= 128.
|
||||
#[cfg(feature = "bitshuffle")]
|
||||
fn default_block_size(es: usize) -> usize {
|
||||
((8192 / es) / 8 * 8).max(128)
|
||||
}
|
||||
|
||||
#[cfg(feature = "bitshuffle")]
|
||||
fn err(msg: &str) -> FormatError {
|
||||
FormatError::DecompressionError(format!("bitshuffle: {msg}"))
|
||||
}
|
||||
|
||||
/// `cd_values[4]`: the compression bitshuffle applies after the transpose.
|
||||
#[cfg(feature = "bitshuffle")]
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum Codec {
|
||||
None,
|
||||
Lz4,
|
||||
Zstd,
|
||||
}
|
||||
|
||||
#[cfg(feature = "bitshuffle")]
|
||||
fn codec(cd: &[u32]) -> Result<Codec, FormatError> {
|
||||
match cd.get(4).copied().unwrap_or(0) {
|
||||
0 => Ok(Codec::None),
|
||||
2 => Ok(Codec::Lz4),
|
||||
3 => Ok(Codec::Zstd),
|
||||
other => Err(FormatError::FilterError(format!(
|
||||
"bitshuffle: unknown compression {other}"
|
||||
))),
|
||||
}
|
||||
}
|
||||
|
||||
/// The element counts of the transposed blocks for `size` elements.
|
||||
#[cfg(feature = "bitshuffle")]
|
||||
fn blocks(size: usize, block: usize) -> impl Iterator<Item = usize> {
|
||||
let full = size / block;
|
||||
let last = (size % block) / 8 * 8;
|
||||
core::iter::repeat_n(block, full).chain((last > 0).then_some(last))
|
||||
}
|
||||
|
||||
/// Decode a bitshuffle-filtered chunk.
|
||||
#[cfg(feature = "bitshuffle")]
|
||||
pub(crate) fn bitshuffle_decode(
|
||||
input: &[u8],
|
||||
ctx: &FilterContext<'_>,
|
||||
) -> Result<Vec<u8>, FormatError> {
|
||||
let cd = ctx.client_data();
|
||||
let es = match cd.get(2) {
|
||||
Some(&e) if e != 0 => e as usize,
|
||||
_ => return Err(err("missing element size")),
|
||||
};
|
||||
let codec = codec(cd)?;
|
||||
let limit = ctx.output_limit();
|
||||
if codec == Codec::None {
|
||||
if input.len() > limit {
|
||||
return Err(err("output exceeds the chunk size"));
|
||||
}
|
||||
let block = match cd.get(3) {
|
||||
Some(&b) if b != 0 => b as usize,
|
||||
_ => default_block_size(es),
|
||||
};
|
||||
if !block.is_multiple_of(8) {
|
||||
return Err(err("block size is not a multiple of 8"));
|
||||
}
|
||||
if !input.len().is_multiple_of(es) {
|
||||
return Err(err("chunk is not a whole number of elements"));
|
||||
}
|
||||
let size = input.len() / es;
|
||||
let mut out = vec![0u8; input.len()];
|
||||
let mut pos = 0;
|
||||
for n in blocks(size, block) {
|
||||
let bytes = n * es;
|
||||
bitunshuffle_block(&input[pos..pos + bytes], &mut out[pos..pos + bytes], n, es);
|
||||
pos += bytes;
|
||||
}
|
||||
out[pos..].copy_from_slice(&input[pos..]);
|
||||
return Ok(out);
|
||||
}
|
||||
|
||||
let header = input.get(..12).ok_or_else(|| err("truncated header"))?;
|
||||
let total = u64::from_be_bytes(header[..8].try_into().unwrap());
|
||||
let block_bytes = u32::from_be_bytes(header[8..12].try_into().unwrap()) as usize;
|
||||
let total = usize::try_from(total)
|
||||
.ok()
|
||||
.filter(|&t| t <= limit)
|
||||
.ok_or_else(|| err("decoded size exceeds the chunk size"))?;
|
||||
if !total.is_multiple_of(es) {
|
||||
return Err(err("chunk is not a whole number of elements"));
|
||||
}
|
||||
if block_bytes == 0 || !block_bytes.is_multiple_of(es) {
|
||||
return Err(err("bad block size"));
|
||||
}
|
||||
let block = block_bytes / es;
|
||||
if !block.is_multiple_of(8) {
|
||||
return Err(err("block size is not a multiple of 8"));
|
||||
}
|
||||
let size = total / es;
|
||||
let mut out = vec![0u8; total];
|
||||
let mut tmp = vec![0u8; block_bytes.min(total)];
|
||||
let mut ip = 12usize;
|
||||
let mut op = 0usize;
|
||||
let mut zstd = None;
|
||||
for n in blocks(size, block) {
|
||||
let bytes = n * es;
|
||||
let len = input
|
||||
.get(ip..ip + 4)
|
||||
.map(|b| u32::from_be_bytes(b.try_into().unwrap()) as usize)
|
||||
.ok_or_else(|| err("truncated block header"))?;
|
||||
ip += 4;
|
||||
let comp = input
|
||||
.get(ip..ip.saturating_add(len))
|
||||
.ok_or_else(|| err("truncated block"))?;
|
||||
ip += len;
|
||||
let dst = &mut tmp[..bytes];
|
||||
let got = match codec {
|
||||
Codec::Lz4 => lz4_flex::block::decompress_into(comp, dst)
|
||||
.map_err(|e| err(&format!("lz4: {e}")))?,
|
||||
Codec::Zstd => zstd_decode_into(
|
||||
zstd.get_or_insert_with(ruzstd::decoding::FrameDecoder::new),
|
||||
comp,
|
||||
dst,
|
||||
)?,
|
||||
Codec::None => unreachable!(),
|
||||
};
|
||||
if got != bytes {
|
||||
return Err(err("block decoded to the wrong size"));
|
||||
}
|
||||
bitunshuffle_block(dst, &mut out[op..op + bytes], n, es);
|
||||
op += bytes;
|
||||
}
|
||||
let tail = total - op;
|
||||
let rest = input
|
||||
.get(ip..ip + tail)
|
||||
.ok_or_else(|| err("truncated trailing elements"))?;
|
||||
out[op..].copy_from_slice(rest);
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Decode Zstandard frames into exactly `dst`, failing if they hold more.
|
||||
#[cfg(feature = "bitshuffle")]
|
||||
pub(crate) fn zstd_decode_into(
|
||||
decoder: &mut ruzstd::decoding::FrameDecoder,
|
||||
frames: &[u8],
|
||||
dst: &mut [u8],
|
||||
) -> Result<usize, FormatError> {
|
||||
decoder
|
||||
.decode_all(frames, dst)
|
||||
.map_err(|e| FormatError::DecompressionError(format!("zstd: {e}")))
|
||||
}
|
||||
|
||||
/// Compress with ruzstd. It implements one level (roughly zstd's level 1),
|
||||
/// so the requested level only matters to other encoders.
|
||||
#[cfg(feature = "bitshuffle")]
|
||||
pub(crate) fn zstd_encode(data: &[u8]) -> Vec<u8> {
|
||||
ruzstd::encoding::compress_to_vec(data, ruzstd::encoding::CompressionLevel::Fastest)
|
||||
}
|
||||
|
||||
/// Encode a chunk with the bitshuffle filter.
|
||||
#[cfg(feature = "bitshuffle")]
|
||||
pub(crate) fn bitshuffle_encode(
|
||||
input: &[u8],
|
||||
ctx: &FilterContext<'_>,
|
||||
) -> Result<Vec<u8>, FormatError> {
|
||||
let cd = ctx.client_data();
|
||||
let es = match cd.get(2) {
|
||||
Some(&e) if e != 0 => e as usize,
|
||||
_ => ctx.element_size.max(1),
|
||||
};
|
||||
let codec = codec(cd)?;
|
||||
let block = match cd.get(3) {
|
||||
Some(&b) if b != 0 => b as usize,
|
||||
_ => default_block_size(es),
|
||||
};
|
||||
let cerr = |m: &str| FormatError::CompressionError(format!("bitshuffle: {m}"));
|
||||
if !block.is_multiple_of(8) {
|
||||
return Err(cerr("block size is not a multiple of 8"));
|
||||
}
|
||||
if !input.len().is_multiple_of(es) {
|
||||
return Err(cerr("chunk is not a whole number of elements"));
|
||||
}
|
||||
let size = input.len() / es;
|
||||
let mut out = Vec::with_capacity(input.len() + 12 + input.len() / 64);
|
||||
if codec != Codec::None {
|
||||
out.extend_from_slice(&(input.len() as u64).to_be_bytes());
|
||||
let block_bytes =
|
||||
u32::try_from(block * es).map_err(|_| cerr("block size does not fit in 32 bits"))?;
|
||||
out.extend_from_slice(&block_bytes.to_be_bytes());
|
||||
}
|
||||
let mut tmp = vec![0u8; (block * es).min(input.len())];
|
||||
let mut pos = 0;
|
||||
for n in blocks(size, block) {
|
||||
let bytes = n * es;
|
||||
let dst = &mut tmp[..bytes];
|
||||
bitshuffle_block(&input[pos..pos + bytes], dst, n, es);
|
||||
match codec {
|
||||
Codec::None => out.extend_from_slice(dst),
|
||||
Codec::Lz4 | Codec::Zstd => {
|
||||
let comp = if codec == Codec::Lz4 {
|
||||
lz4_flex::block::compress(dst)
|
||||
} else {
|
||||
zstd_encode(dst)
|
||||
};
|
||||
out.extend_from_slice(&(comp.len() as u32).to_be_bytes());
|
||||
out.extend_from_slice(&comp);
|
||||
}
|
||||
}
|
||||
pos += bytes;
|
||||
}
|
||||
out.extend_from_slice(&input[pos..]);
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// The definition, one bit at a time.
|
||||
fn naive(input: &[u8], n: usize, es: usize) -> Vec<u8> {
|
||||
let mut out = vec![0u8; n * es];
|
||||
for i in 0..n {
|
||||
for j in 0..es {
|
||||
for k in 0..8 {
|
||||
if input[i * es + j] >> k & 1 == 1 {
|
||||
let p = (8 * j + k) * n + i;
|
||||
out[p / 8] |= 1 << (p % 8);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transpose_matches_the_definition_and_inverts() {
|
||||
for (n, es) in [(8, 1), (16, 2), (24, 4), (128, 8), (64, 3), (8, 16)] {
|
||||
let input: Vec<u8> = (0..n * es)
|
||||
.map(|i| (i as u32).wrapping_mul(2_654_435_761).rotate_left(7) as u8)
|
||||
.collect();
|
||||
let mut out = vec![0u8; n * es];
|
||||
bitshuffle_block(&input, &mut out, n, es);
|
||||
assert_eq!(out, naive(&input, n, es), "n={n} es={es}");
|
||||
let mut back = vec![0u8; n * es];
|
||||
bitunshuffle_block(&out, &mut back, n, es);
|
||||
assert_eq!(back, input);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "bitshuffle")]
|
||||
fn ctx_for(cd: Vec<u32>) -> crate::filter_pipeline::FilterDescription {
|
||||
crate::filter_pipeline::FilterDescription {
|
||||
filter_id: crate::filter_pipeline::FILTER_BITSHUFFLE,
|
||||
name: None,
|
||||
flags: 0,
|
||||
client_data: cd,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "bitshuffle")]
|
||||
#[test]
|
||||
fn filter_round_trips_every_mode() {
|
||||
for es in [1usize, 2, 4, 8] {
|
||||
for n in [0usize, 1, 7, 8, 100, 1000, 5003] {
|
||||
let data: Vec<u8> = (0..n * es)
|
||||
.map(|i| (i % 97) as u8 ^ (i / 300) as u8)
|
||||
.collect();
|
||||
for (comp, block) in [(0, 0), (0, 16), (2, 0), (2, 64), (3, 0), (3, 1024)] {
|
||||
let f = ctx_for(vec![0, 4, es as u32, block, comp]);
|
||||
let ctx = FilterContext {
|
||||
filter: &f,
|
||||
element_size: es,
|
||||
max_output: data.len(),
|
||||
};
|
||||
let enc = bitshuffle_encode(&data, &ctx).unwrap();
|
||||
let dec = bitshuffle_decode(&enc, &ctx).unwrap();
|
||||
assert_eq!(dec, data, "es={es} n={n} comp={comp} block={block}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "bitshuffle")]
|
||||
#[test]
|
||||
fn rejects_oversized_and_truncated_chunks() {
|
||||
let data = vec![5u8; 4096];
|
||||
let f = ctx_for(vec![0, 4, 4, 0, 2]);
|
||||
let mut ctx = FilterContext {
|
||||
filter: &f,
|
||||
element_size: 4,
|
||||
max_output: data.len(),
|
||||
};
|
||||
let enc = bitshuffle_encode(&data, &ctx).unwrap();
|
||||
assert!(bitshuffle_decode(&enc[..enc.len() - 1], &ctx).is_err());
|
||||
ctx.max_output = 100;
|
||||
assert!(bitshuffle_decode(&enc, &ctx).is_err());
|
||||
}
|
||||
}
|
||||
@@ -73,6 +73,8 @@ pub mod fill_value;
|
||||
pub mod filter_pipeline;
|
||||
pub mod filter_registry;
|
||||
pub mod filters;
|
||||
#[cfg(feature = "bitshuffle")]
|
||||
mod filters_bitshuffle;
|
||||
#[cfg(feature = "lzf")]
|
||||
pub mod filters_lzf;
|
||||
mod filters_szip;
|
||||
|
||||
@@ -749,6 +749,20 @@ impl DatasetBuilder {
|
||||
self.with_plugin_filter(crate::chunked_write::PluginFilter::Lzf)
|
||||
}
|
||||
|
||||
/// Enable bitshuffle (filter 32008) with `compression` after the bit
|
||||
/// transpose, in bitshuffle's default block size. Implies chunked
|
||||
/// storage; no byte shuffle is added. Requires the `bitshuffle` cargo
|
||||
/// feature.
|
||||
pub fn with_bitshuffle(
|
||||
&mut self,
|
||||
compression: crate::chunked_write::BitshuffleCompression,
|
||||
) -> &mut Self {
|
||||
self.with_plugin_filter(crate::chunked_write::PluginFilter::Bitshuffle {
|
||||
block_size: 0,
|
||||
compression,
|
||||
})
|
||||
}
|
||||
|
||||
/// Enable Pcodec lossless numerical compression (private clawhdf5 filter
|
||||
/// ID 480).
|
||||
///
|
||||
|
||||
@@ -44,6 +44,7 @@ pcodec = ["clawhdf5-format/pcodec"]
|
||||
# Plugin filters, pure Rust (no C). LZF (32000) is h5py's built-in
|
||||
# compression; it has no dependencies, so it is on by default.
|
||||
lzf = ["clawhdf5-format/lzf"]
|
||||
bitshuffle = ["clawhdf5-format/bitshuffle"]
|
||||
# Dataset::verify_provenance() — recompute a dataset's SHA-256 and compare
|
||||
# against its stored _provenance_sha256 attribute. On by default, matching
|
||||
# clawhdf5-format's own default-on `provenance` feature.
|
||||
|
||||
@@ -234,6 +234,47 @@ fn check_ours_read_by_h5py(
|
||||
assert_eq!(out, "OK", "{tag}: h5py could not read our output");
|
||||
}
|
||||
|
||||
#[cfg(feature = "bitshuffle")]
|
||||
#[test]
|
||||
fn bitshuffle_written_by_hdf5plugin_reads_exactly() {
|
||||
if !have_python("h5py, hdf5plugin") {
|
||||
return;
|
||||
}
|
||||
check_h5py_written(
|
||||
"bitshuffle",
|
||||
r#"[('none', hdf5plugin.Bitshuffle(cname='none')),
|
||||
('lz4', hdf5plugin.Bitshuffle(cname='lz4')),
|
||||
('lz4 nelems=16', hdf5plugin.Bitshuffle(nelems=16, cname='lz4')),
|
||||
('none nelems=64', hdf5plugin.Bitshuffle(nelems=64, cname='none')),
|
||||
('zstd', hdf5plugin.Bitshuffle(cname='zstd')),
|
||||
('zstd clevel=19 nelems=2048', hdf5plugin.Bitshuffle(nelems=2048, cname='zstd', clevel=19))]"#,
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "bitshuffle")]
|
||||
#[test]
|
||||
fn bitshuffle_written_by_clawhdf5_reads_in_hdf5plugin() {
|
||||
use clawhdf5_format::chunked_write::{BitshuffleCompression, PluginFilter};
|
||||
if !have_python("h5py, hdf5plugin") {
|
||||
return;
|
||||
}
|
||||
for (tag, compression) in [
|
||||
("bshuf_none", BitshuffleCompression::None),
|
||||
("bshuf_lz4", BitshuffleCompression::Lz4),
|
||||
("bshuf_zstd", BitshuffleCompression::Zstd { level: 3 }),
|
||||
] {
|
||||
check_ours_read_by_h5py(tag, 32008, "bitshuffle", |ds| {
|
||||
ds.with_bitshuffle(compression);
|
||||
});
|
||||
check_ours_read_by_h5py(tag, 32008, "bitshuffle", |ds| {
|
||||
ds.with_plugin_filter(PluginFilter::Bitshuffle {
|
||||
block_size: 40,
|
||||
compression,
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "lzf")]
|
||||
#[test]
|
||||
fn lzf_written_by_h5py_reads_exactly() {
|
||||
|
||||
Reference in New Issue
Block a user