diff --git a/crates/clawhdf5-format/Cargo.toml b/crates/clawhdf5-format/Cargo.toml index a88b734..4eb4c21 100644 --- a/crates/clawhdf5-format/Cargo.toml +++ b/crates/clawhdf5-format/Cargo.toml @@ -37,7 +37,7 @@ harness = false # Deflate backend: `zlib-rs` (pure Rust) by default. `fast-deflate` selects # zlib-ng instead (C, built with cmake); flate2 prefers a C zlib whenever one # is enabled, so turning it on anywhere in the build overrides the default. -default = ["std", "checksum", "deflate", "provenance", "zlib-rs", "system-zlib-decompress"] +default = ["std", "checksum", "deflate", "provenance", "zlib-rs", "system-zlib-decompress", "lzf"] std = [] checksum = [] deflate = ["flate2"] @@ -56,6 +56,9 @@ zstd = ["dep:zstd"] blake3_hash = ["blake3"] szip = ["libaec-sys"] pcodec = ["dep:pco"] +# Plugin filters, pure Rust. LZF (32000) is h5py's built-in compression; it +# has no dependencies, so it is on by default. +lzf = [] [[bench]] name = "parallel_decompress_bench" diff --git a/crates/clawhdf5-format/src/chunked_write.rs b/crates/clawhdf5-format/src/chunked_write.rs index ff5081e..4e9b851 100644 --- a/crates/clawhdf5-format/src/chunked_write.rs +++ b/crates/clawhdf5-format/src/chunked_write.rs @@ -12,7 +12,7 @@ use crate::chunk_grid::ChunkGrid; use crate::ea_writer; use crate::error::FormatError; use crate::filter_pipeline::{ - FILTER_DEFLATE, FILTER_FLETCHER32, FILTER_LZ4, FILTER_PCODEC, FILTER_PCODEC_NAME, + FILTER_DEFLATE, FILTER_FLETCHER32, FILTER_LZ4, FILTER_LZF, FILTER_PCODEC, FILTER_PCODEC_NAME, FILTER_SHUFFLE, FILTER_ZSTD, FilterDescription, FilterPipeline, }; use crate::filters::compress_chunk; @@ -48,6 +48,45 @@ pub struct ChunkOptions { /// Pcodec lossless numerical compression. Private, unregistered filter /// ID [`FILTER_PCODEC`] (480): only clawhdf5 can read it. pub pcodec: bool, + /// A plugin compression filter (LZF, ...). Takes priority over the + /// codecs above. Each needs its cargo feature to be written. + pub plugin: Option, +} + +/// A compression filter from the common HDF5 plugin set, written in the +/// format the libhdf5 plugin (h5py / hdf5plugin) reads. +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub enum PluginFilter { + /// LZF (filter 32000), h5py's built-in `compression="lzf"`. Needs the + /// `lzf` feature. + Lzf, +} + +impl PluginFilter { + /// Whether the filter reorders bytes itself, so the automatic shuffle + /// pre-filter would only get in its way. + fn shuffles_itself(&self) -> bool { + match self { + PluginFilter::Lzf => false, + } + } + + /// The pipeline entry for this filter. `chunk_bytes` is one chunk's + /// uncompressed size (0 if unknown). + fn description(&self, _element_size: u32, chunk_bytes: u32) -> FilterDescription { + match self { + // h5py's lzf_set_local: filter version, liblzf version, chunk + // size in bytes. Optional, as h5py flags it: a chunk the filter + // cannot shrink may then be stored unfiltered. + PluginFilter::Lzf => FilterDescription { + filter_id: FILTER_LZF, + name: Some("lzf".into()), + flags: 1, + client_data: vec![4, 0x0105, chunk_bytes], + }, + } + } } /// Largest chunk the automatic choice produces, in bytes. @@ -92,14 +131,33 @@ impl ChunkOptions { || self.lz4 || self.zstd_level.is_some() || self.pcodec + || self.plugin.is_some() } /// Build a FilterPipeline from the options. pub fn build_pipeline(&self, element_size: u32) -> Option { + self.build_pipeline_for_chunk(element_size, 0) + } + + /// Build a FilterPipeline for chunks of `chunk_bytes` uncompressed bytes + /// (0 if unknown). Some plugin filters record the chunk size in their + /// client data. + pub fn build_pipeline_for_chunk( + &self, + element_size: u32, + chunk_bytes: u32, + ) -> Option { let mut filters = Vec::new(); - let has_compression = - self.deflate_level.is_some() || self.zstd_level.is_some() || self.lz4 || self.pcodec; + let plugin_shuffles = self + .plugin + .as_ref() + .is_some_and(PluginFilter::shuffles_itself); + let has_compression = self.deflate_level.is_some() + || self.zstd_level.is_some() + || self.lz4 + || self.pcodec + || (self.plugin.is_some() && !plugin_shuffles); // Shuffle before compression. Applied if explicitly requested OR if compression // is active and the caller hasn't disabled it — matches h5py default behavior @@ -113,8 +171,11 @@ impl ChunkOptions { }); } - // Compression filters (mutually exclusive, priority: pcodec > zstd > lz4 > deflate) - if self.pcodec { + // Compression filters (mutually exclusive, priority: plugin > pcodec > + // zstd > lz4 > deflate) + if let Some(plugin) = &self.plugin { + filters.push(plugin.description(element_size, chunk_bytes)); + } else if self.pcodec { filters.push(FilterDescription { filter_id: FILTER_PCODEC, name: Some(FILTER_PCODEC_NAME.into()), @@ -675,7 +736,12 @@ pub fn precompress_chunks( element_size: usize, options: &ChunkOptions, ) -> Result { - let pipeline = options.build_pipeline(element_size as u32); + let chunk_bytes = chunk_dims + .iter() + .try_fold(element_size as u64, |acc, &d| acc.checked_mul(d)) + .and_then(|b| u32::try_from(b).ok()) + .unwrap_or(0); + let pipeline = options.build_pipeline_for_chunk(element_size as u32, chunk_bytes); let has_filters = pipeline.is_some(); let pipeline_message = pipeline.as_ref().map(|pl| pl.serialize()); @@ -1569,6 +1635,20 @@ mod tests { assert_eq!(pl.filters[1].client_data, vec![3]); } + #[test] + fn chunk_options_pipeline_lzf() { + let options = ChunkOptions { + plugin: Some(PluginFilter::Lzf), + ..Default::default() + }; + assert!(options.is_chunked()); + let pl = options.build_pipeline_for_chunk(8, 800).unwrap(); + assert_eq!(pl.filters.len(), 2); + assert_eq!(pl.filters[0].filter_id, FILTER_SHUFFLE); + assert_eq!(pl.filters[1].filter_id, FILTER_LZF); + assert_eq!(pl.filters[1].client_data, vec![4, 0x0105, 800]); + } + #[test] fn chunk_options_zstd_priority_over_deflate() { let options = ChunkOptions { diff --git a/crates/clawhdf5-format/src/filter_pipeline.rs b/crates/clawhdf5-format/src/filter_pipeline.rs index 74bc727..13ed69f 100644 --- a/crates/clawhdf5-format/src/filter_pipeline.rs +++ b/crates/clawhdf5-format/src/filter_pipeline.rs @@ -19,6 +19,18 @@ pub const FILTER_SCALEOFFSET: u16 = 6; pub const FILTER_LZ4: u16 = 32004; /// Zstandard compression. pub const FILTER_ZSTD: u16 = 32015; +/// bzip2 (registered by PyTables; hdf5plugin's `BZip2`). +pub const FILTER_BZIP2: u16 = 307; +/// LZF — h5py's built-in `compression="lzf"`. +pub const FILTER_LZF: u16 = 32000; +/// Blosc 1 (hdf5-blosc; hdf5plugin's `Blosc`). +pub const FILTER_BLOSC: u16 = 32001; +/// Bitshuffle, optionally with LZ4 or Zstandard (hdf5plugin's `Bitshuffle`). +pub const FILTER_BITSHUFFLE: u16 = 32008; +/// ZFP lossy floating-point compression (hdf5plugin's `Zfp`). Not supported. +pub const FILTER_ZFP: u16 = 32013; +/// Blosc 2 (hdf5plugin's `Blosc2`). +pub const FILTER_BLOSC2: u16 = 32026; /// Pcodec lossless numerical codec — a **private, unregistered** clawhdf5 /// filter. Pcodec has no ID in the HDF Group's filter registry (checked /// 2026-09-25, `hdf5_plugins/docs/RegisteredFilterPlugins.md`), so it uses an diff --git a/crates/clawhdf5-format/src/filters.rs b/crates/clawhdf5-format/src/filters.rs index e833c7c..237ceb5 100644 --- a/crates/clawhdf5-format/src/filters.rs +++ b/crates/clawhdf5-format/src/filters.rs @@ -192,6 +192,13 @@ pub(crate) static BUILTIN_FILTERS: &[BuiltinFilter] = &[ decode: |d, c| pcodec_decompress(d, c.element_size, c.max_output), encode: Some(|d, c| pcodec_compress(d, c.element_size)), }, + #[cfg(feature = "lzf")] + BuiltinFilter { + id: crate::filter_pipeline::FILTER_LZF, + name: "lzf", + decode: crate::filters_lzf::lzf_decode, + encode: Some(crate::filters_lzf::lzf_encode), + }, #[cfg(feature = "lz4")] BuiltinFilter { id: FILTER_LZ4, @@ -1702,7 +1709,7 @@ mod tests { // An unsupported filter is fine when the chunk skipped it. let unknown = FilterPipeline { version: 2, - filters: vec![filter(32000), filter(FILTER_DEFLATE)], + filters: vec![filter(32013), filter(FILTER_DEFLATE)], }; assert_eq!( decompress_chunk_masked(&deflated, &unknown, n, 8, 0b01).unwrap(), diff --git a/crates/clawhdf5-format/src/filters_lzf.rs b/crates/clawhdf5-format/src/filters_lzf.rs new file mode 100644 index 0000000..5657339 --- /dev/null +++ b/crates/clawhdf5-format/src/filters_lzf.rs @@ -0,0 +1,232 @@ +//! LZF (HDF5 filter 32000) — h5py's built-in compression filter +//! (`compression="lzf"`), in pure Rust. +//! +//! The chunk is one raw LZF stream (liblzf 3.x format, no header). The +//! stream is a sequence of instructions, each starting with a control byte: +//! +//! * `000LLLLL` — a literal run: the next `L + 1` bytes (1..=32) are copied. +//! * `LLLOOOOO [E] OOOOOOOO` — a back reference: copy `len + 2` bytes from +//! `distance` bytes back, where `len` is the top three bits (1..=6), or +//! `7 + E` when they are all ones, and `distance` is the 13-bit offset +//! (high five bits in the control byte, low eight in the last byte) plus 1. +//! +//! h5py's filter (`lzf_filter.c`) records the chunk's size in bytes in +//! `cd_values[2]` (slots 0 and 1 hold the filter and liblzf versions) and +//! sizes its output buffer from it. + +#[cfg(not(feature = "std"))] +extern crate alloc; +#[cfg(not(feature = "std"))] +use alloc::{format, vec, vec::Vec}; + +use crate::error::FormatError; +use crate::filter_registry::FilterContext; + +/// `H5PY_FILTER_LZF_VERSION`, written to `cd_values[0]`. +pub const LZF_FILTER_VERSION: u32 = 4; +/// `LZF_VERSION` (liblzf 1.5), written to `cd_values[1]`. +pub const LZF_API_VERSION: u32 = 0x0105; + +const MAX_LITERAL: usize = 32; +const MAX_OFFSET: usize = 1 << 13; +const MAX_REF: usize = (1 << 8) + (1 << 3); +const HASH_LOG: u32 = 14; + +fn err(msg: &str) -> FormatError { + FormatError::DecompressionError(format!("lzf: {msg}")) +} + +/// Decode an LZF-filtered chunk. +pub(crate) fn lzf_decode(input: &[u8], ctx: &FilterContext<'_>) -> Result, FormatError> { + let limit = ctx.output_limit(); + let hint = match ctx.client_data().get(2) { + Some(&n) if n != 0 => n as usize, + _ => input.len().saturating_mul(2), + }; + lzf_decompress(input, hint.min(limit), limit) +} + +/// Decompress a raw LZF stream, refusing to produce more than `limit` bytes. +pub fn lzf_decompress( + input: &[u8], + size_hint: usize, + limit: usize, +) -> Result, FormatError> { + let mut out: Vec = Vec::new(); + out.try_reserve(size_hint) + .map_err(|_| err("cannot allocate the output buffer"))?; + let mut ip = 0usize; + while ip < input.len() { + let ctrl = input[ip] as usize; + ip += 1; + if ctrl < 32 { + let run = ctrl + 1; + let lit = input + .get(ip..ip + run) + .ok_or_else(|| err("literal run past the end of the input"))?; + if out.len() + run > limit { + return Err(err("output exceeds the chunk size")); + } + out.extend_from_slice(lit); + ip += run; + } else { + let mut len = ctrl >> 5; + if len == 7 { + len += *input + .get(ip) + .ok_or_else(|| err("truncated back reference"))? + as usize; + ip += 1; + } + let low = *input + .get(ip) + .ok_or_else(|| err("truncated back reference"))? as usize; + ip += 1; + let distance = ((ctrl & 0x1f) << 8) + low + 1; + let len = len + 2; + if distance > out.len() { + return Err(err("back reference before the start of the output")); + } + if out.len() + len > limit { + return Err(err("output exceeds the chunk size")); + } + let start = out.len() - distance; + if distance >= len { + out.extend_from_within(start..start + len); + } else { + // Overlapping copy: repeats the last `distance` bytes. + for k in 0..len { + let b = out[start + k]; + out.push(b); + } + } + } + } + Ok(out) +} + +/// Encode a chunk with the LZF filter. +pub(crate) fn lzf_encode(input: &[u8], _ctx: &FilterContext<'_>) -> Result, FormatError> { + Ok(lzf_compress(input)) +} + +fn hash3(b: &[u8]) -> usize { + let v = (u32::from(b[0]) << 16) | (u32::from(b[1]) << 8) | u32::from(b[2]); + (v.wrapping_mul(2_654_435_761) >> (32 - HASH_LOG)) as usize +} + +fn flush_literals(out: &mut Vec, lit: &[u8]) { + for run in lit.chunks(MAX_LITERAL) { + out.push((run.len() - 1) as u8); + out.extend_from_slice(run); + } +} + +/// Compress `input` into a raw LZF stream any liblzf decoder reads. +/// +/// Incompressible input grows by one byte per 32. (h5py's own filter gives +/// up on such a chunk and stores it unfiltered; storing the slightly larger +/// stream is equally readable.) +pub fn lzf_compress(input: &[u8]) -> Vec { + let n = input.len(); + let mut out = Vec::with_capacity(n + n / MAX_LITERAL + 1); + let mut table = vec![0u32; 1 << HASH_LOG]; + let mut lit_start = 0usize; + let mut i = 0usize; + while i + 2 < n { + let h = hash3(&input[i..]); + let cand = table[h] as usize; + table[h] = (i + 1) as u32; + if cand != 0 { + let r = cand - 1; + let distance = i - r; + if distance <= MAX_OFFSET && input[r..r + 3] == input[i..i + 3] { + let max_len = (n - i).min(MAX_REF); + let mut len = 3; + while len < max_len && input[r + len] == input[i + len] { + len += 1; + } + flush_literals(&mut out, &input[lit_start..i]); + let code = len - 2; + let off = distance - 1; + if code < 7 { + out.push(((code << 5) | (off >> 8)) as u8); + } else { + out.push(((7 << 5) | (off >> 8)) as u8); + out.push((code - 7) as u8); + } + out.push((off & 0xff) as u8); + // Index the positions the match covered so later data can + // refer back into it. + let end = i + len; + let mut j = i + 1; + while j < end && j + 2 < n { + table[hash3(&input[j..])] = (j + 1) as u32; + j += 1; + } + i = end; + lit_start = i; + continue; + } + } + i += 1; + } + flush_literals(&mut out, &input[lit_start..]); + out +} + +#[cfg(test)] +mod tests { + use super::*; + + fn round_trip(data: &[u8]) { + let c = lzf_compress(data); + assert_eq!(lzf_decompress(&c, data.len(), data.len()).unwrap(), data); + } + + #[test] + fn round_trips() { + round_trip(b""); + round_trip(b"a"); + round_trip(b"abcabcabcabcabcabcabcabcabcabcabcabc"); + round_trip(&[7u8; 10_000]); + let noise: Vec = (0..70_000u32) + .map(|i| (i.wrapping_mul(2_654_435_761) >> 13) as u8) + .collect(); + round_trip(&noise); + let ramp: Vec = (0..100_000u32) + .flat_map(|i| (i % 1000).to_le_bytes()) + .collect(); + round_trip(&ramp); + } + + #[test] + fn compresses_repetitive_data() { + let data = [42u8; 4096]; + assert!(lzf_compress(&data).len() < 100); + } + + /// The chunk h5py 3.16's bundled liblzf writes for + /// `b"hello hello hello hello"` (read back with `read_direct_chunk`): a + /// 7-byte literal, a 14-byte back reference 6 bytes back (extended + /// length), and a 2-byte literal. + #[test] + fn decodes_liblzf_output() { + let stream = b"\x06hello h\xe0\x05\x05\x01lo"; + assert_eq!( + lzf_decompress(stream, 23, 23).unwrap(), + b"hello hello hello hello" + ); + } + + #[test] + fn rejects_corrupt_streams() { + // Back reference before the start. + assert!(lzf_decompress(&[0x20, 0x00], 10, 10).is_err()); + // Literal run past the end. + assert!(lzf_decompress(&[0x05, 1, 2], 10, 10).is_err()); + // Output over the limit. + let c = lzf_compress(&[1u8; 100]); + assert!(lzf_decompress(&c, 10, 99).is_err()); + } +} diff --git a/crates/clawhdf5-format/src/lib.rs b/crates/clawhdf5-format/src/lib.rs index 74573c0..c717d89 100644 --- a/crates/clawhdf5-format/src/lib.rs +++ b/crates/clawhdf5-format/src/lib.rs @@ -73,6 +73,8 @@ pub mod fill_value; pub mod filter_pipeline; pub mod filter_registry; pub mod filters; +#[cfg(feature = "lzf")] +pub mod filters_lzf; mod filters_szip; pub mod fixed_array; pub mod float16; diff --git a/crates/clawhdf5-format/src/type_builders.rs b/crates/clawhdf5-format/src/type_builders.rs index 9e46e66..c49ccc7 100644 --- a/crates/clawhdf5-format/src/type_builders.rs +++ b/crates/clawhdf5-format/src/type_builders.rs @@ -731,6 +731,24 @@ impl DatasetBuilder { self } + /// Compress with a plugin filter ([`PluginFilter`]), in the format the + /// libhdf5 plugin reads (h5py, hdf5plugin). Implies chunked storage. + /// Each filter needs its cargo feature (`lzf`, ...); writing fails with + /// `UnsupportedFilter` without it. + /// + /// [`PluginFilter`]: crate::chunked_write::PluginFilter + pub fn with_plugin_filter(&mut self, filter: crate::chunked_write::PluginFilter) -> &mut Self { + self.chunk_options.plugin = Some(filter); + self + } + + /// Enable LZF compression (filter 32000) — h5py's built-in + /// `compression="lzf"`. Implies chunked storage; shuffle is applied + /// first unless `.without_shuffle()`. Requires the `lzf` cargo feature. + pub fn with_lzf(&mut self) -> &mut Self { + self.with_plugin_filter(crate::chunked_write::PluginFilter::Lzf) + } + /// Enable Pcodec lossless numerical compression (private clawhdf5 filter /// ID 480). /// diff --git a/crates/clawhdf5/Cargo.toml b/crates/clawhdf5/Cargo.toml index 2e2b4b4..f4c6ee2 100644 --- a/crates/clawhdf5/Cargo.toml +++ b/crates/clawhdf5/Cargo.toml @@ -31,7 +31,7 @@ name = "parallel_bench" harness = false [features] -default = ["mmap", "provenance"] +default = ["mmap", "provenance", "lzf"] mmap = ["clawhdf5-io/mmap"] parallel = ["clawhdf5-format/parallel", "rayon"] # zlib-ng (C, needs cmake) instead of the default pure-Rust zlib-rs. @@ -41,6 +41,9 @@ zstd = ["clawhdf5-format/zstd"] blake3_hash = ["clawhdf5-format/blake3_hash"] lz4 = ["clawhdf5-format/lz4"] pcodec = ["clawhdf5-format/pcodec"] +# Plugin filters, pure Rust (no C). LZF (32000) is h5py's built-in +# compression; it has no dependencies, so it is on by default. +lzf = ["clawhdf5-format/lzf"] # Dataset::verify_provenance() — recompute a dataset's SHA-256 and compare # against its stored _provenance_sha256 attribute. On by default, matching # clawhdf5-format's own default-on `provenance` feature. diff --git a/crates/clawhdf5/tests/plugin_filters_interop.rs b/crates/clawhdf5/tests/plugin_filters_interop.rs new file mode 100644 index 0000000..041aff1 --- /dev/null +++ b/crates/clawhdf5/tests/plugin_filters_interop.rs @@ -0,0 +1,263 @@ +//! Plugin filters (LZF, bitshuffle, bzip2, blosc) against libhdf5. +//! +//! Read direction: h5py (with hdf5plugin for everything but LZF, which h5py +//! ships) writes each filter over a matrix of dtypes (1-8 bytes, both byte +//! orders), 1-3 dimensional shapes whose chunks do not divide them (so the +//! edge chunks are partial), compressible and incompressible data, and the +//! filter's own options; every dataset has an unfiltered twin, and clawhdf5 +//! must read the filtered one byte for byte equal to it. +//! +//! Write direction: clawhdf5 writes with its encoder, and h5py must read the +//! values back. +//! +//! Skipped when python3 with h5py (and hdf5plugin) is unavailable, unless +//! `CLAWHDF5_REQUIRE_INTEROP=1`. +#![allow(dead_code)] + +use std::process::Command; + +use clawhdf5::File; +use clawhdf5_format::selection::Selection; + +fn python() -> String { + std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) +} + +fn interop_required() -> bool { + std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1") +} + +fn python_has(modules: &str) -> bool { + Command::new(python()) + .args(["-c", &format!("import {modules}")]) + .output() + .map(|o| o.status.success()) + .unwrap_or(false) +} + +/// Whether the interop test can run; panics instead of skipping when +/// `CLAWHDF5_REQUIRE_INTEROP=1`. +fn have_python(modules: &str) -> bool { + if python_has(modules) { + return true; + } + assert!( + !interop_required(), + "CLAWHDF5_REQUIRE_INTEROP=1 but python3 with {modules} is not available" + ); + eprintln!("SKIP: python3 with {modules} not available"); + false +} + +fn run_python(script: &str, args: &[&str]) -> String { + let output = Command::new(python()) + .arg("-c") + .arg(script) + .args(args) + .output() + .expect("failed to run python"); + if !output.status.success() { + panic!( + "Python script failed:\nSTDOUT: {}\nSTDERR: {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } + String::from_utf8_lossy(&output.stdout).trim().to_string() +} + +/// Writes `f{i}` (filtered) and `r{i}` (unfiltered twin) for every filter +/// setting in `FILTERS` × every case; prints the number of pairs. +const GENERATE: &str = r#" +import sys +import numpy as np, h5py +try: + import hdf5plugin +except ImportError: + hdf5plugin = None +path = sys.argv[1] +FILTERS = eval(sys.argv[2]) +cases = [ + ('i4', (37, 53), (8, 8), 'ramp'), + (' 0); + let file = File::open(&path).unwrap(); + for i in 0..n { + let filtered = file.dataset(&format!("f{i}")).unwrap(); + let case = format!("{:?}", filtered.attrs().unwrap().get("case")); + let got = filtered + .read_selection(&Selection::All) + .unwrap_or_else(|e| panic!("{tag} f{i} {case}: {e}")); + let want = file + .dataset(&format!("r{i}")) + .unwrap() + .read_selection(&Selection::All) + .unwrap(); + assert!(got == want, "{tag} f{i} {case}: data differs"); + } +} + +/// Values the write-direction tests store: row-major, compressible with some +/// variation. +fn ramp_i32(n: usize) -> Vec { + (0..n).map(|i| ((i * 3) % 251) as i32 - 60).collect() +} +fn ramp_f64(n: usize) -> Vec { + (0..n).map(|i| ((i * 3) % 251) as f64 / 7.0).collect() +} +fn ramp_u8(n: usize) -> Vec { + (0..n).map(|i| ((i * 7) % 256) as u8).collect() +} + +/// h5py checks the datasets `write_ours` wrote against the same ramps. +const VERIFY: &str = r#" +import sys +import numpy as np, h5py +try: + import hdf5plugin +except ImportError: + pass +bad = [] +with h5py.File(sys.argv[1], 'r') as f: + for name in f: + ds = f[name] + n = ds.size + k = np.arange(n) + if ds.dtype == np.int32: + want = ((k * 3) % 251 - 60).astype(np.int32) + elif ds.dtype == np.float64: + want = ((k * 3) % 251) / 7.0 + else: + want = ((k * 7) % 256).astype(np.uint8) + got = ds[()].reshape(-1) + if not np.array_equal(got, want): + bad.append(name) + # The dataset really is filtered by the plugin under test. + if int(sys.argv[2]) not in [int(x) for x in ds._filters.keys() if x.isdigit()] \ + and sys.argv[3] not in ds._filters: + bad.append(name + ':filter-missing:' + repr(ds._filters)) +print('OK' if not bad else 'BAD ' + ' '.join(bad)) +"#; + +/// Write i32/f64/u8 datasets (1-D and 2-D, partial edge chunks) with +/// `configure` applying the filter, then have h5py read them back. +fn check_ours_read_by_h5py( + tag: &str, + filter_id: u16, + filter_name: &str, + configure: impl Fn(&mut clawhdf5_format::type_builders::DatasetBuilder), +) { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join(format!("{tag}_ours.h5")); + let mut fb = clawhdf5::FileBuilder::new(); + { + let ds = fb.create_dataset("i32_1d"); + ds.with_i32_data(&ramp_i32(10_000)).with_chunks(&[3000]); + configure(ds); + } + { + let ds = fb.create_dataset("f64_2d"); + ds.with_f64_data(&ramp_f64(37 * 53)) + .with_shape(&[37, 53]) + .with_chunks(&[10, 16]); + configure(ds); + } + { + let ds = fb.create_dataset("u8_1d"); + ds.with_u8_data(&ramp_u8(5000)).with_chunks(&[777]); + configure(ds); + } + { + let ds = fb.create_dataset("f64_big"); + ds.with_f64_data(&ramp_f64(100_000)).with_chunks(&[40_000]); + configure(ds); + } + fb.write(&path).unwrap(); + + // clawhdf5 reads its own output. + let file = File::open(&path).unwrap(); + assert_eq!( + file.dataset("i32_1d").unwrap().read_i32().unwrap(), + ramp_i32(10_000) + ); + assert_eq!( + file.dataset("f64_2d").unwrap().read_f64().unwrap(), + ramp_f64(37 * 53) + ); + + let out = run_python( + VERIFY, + &[path.to_str().unwrap(), &filter_id.to_string(), filter_name], + ); + assert_eq!(out, "OK", "{tag}: h5py could not read our output"); +} + +#[cfg(feature = "lzf")] +#[test] +fn lzf_written_by_h5py_reads_exactly() { + if !have_python("h5py") { + return; + } + check_h5py_written( + "lzf", + r#"[('lzf', dict(compression='lzf')), + ('lzf+shuffle', dict(compression='lzf', shuffle=True)), + ('lzf+shuffle+fletcher32', dict(compression='lzf', shuffle=True, fletcher32=True))]"#, + ); +} + +#[cfg(feature = "lzf")] +#[test] +fn lzf_written_by_clawhdf5_reads_in_h5py() { + if !have_python("h5py") { + return; + } + check_ours_read_by_h5py("lzf", 32000, "lzf", |ds| { + ds.with_lzf(); + }); + check_ours_read_by_h5py("lzf_noshuffle", 32000, "lzf", |ds| { + ds.with_lzf().without_shuffle(); + }); +}