diff --git a/README.md b/README.md index d21462c..5d4c707 100644 --- a/README.md +++ b/README.md @@ -696,7 +696,7 @@ stores keep their setting. Opt out with `float16 = false` or | `fast-checksum` | no | crc32fast-accelerated checksums | | `lz4` | no | LZ4 block compression filter (id 32004) | | `zstd` | no | Zstandard compression filter (id 32015) | -| `pcodec` | no | Pcodec lossless numerical codec (id 32023, via `pco` crate) | +| `pcodec` | no | Pcodec lossless numerical codec (via `pco` crate). Private, unregistered filter id 480: **only clawhdf5 can read these datasets** (h5py/libhdf5 cannot). Files from clawhdf5 <= 2.7.0 used id 32023, which is registered to Granular BitRound; they still read. | | `system-zlib` | no | System zlib backend for deflate (C) | | `blake3_hash` | no | BLAKE3 content hashing for provenance | | `szip` | no | SZIP filter (id 4) via libaec (C, through the internal `libaec-sys` crate) | diff --git a/crates/clawhdf5-format/src/chunked_write.rs b/crates/clawhdf5-format/src/chunked_write.rs index 6816546..aa8bb8f 100644 --- a/crates/clawhdf5-format/src/chunked_write.rs +++ b/crates/clawhdf5-format/src/chunked_write.rs @@ -11,8 +11,8 @@ use crate::chunk_cache::{CACHE_LINE_SIZE, align_to_cache_line}; use crate::ea_writer; use crate::error::FormatError; use crate::filter_pipeline::{ - FILTER_DEFLATE, FILTER_FLETCHER32, FILTER_LZ4, FILTER_PCODEC, FILTER_SHUFFLE, FILTER_ZSTD, - FilterDescription, FilterPipeline, + FILTER_DEFLATE, FILTER_FLETCHER32, FILTER_LZ4, FILTER_PCODEC, FILTER_PCODEC_NAME, + FILTER_SHUFFLE, FILTER_ZSTD, FilterDescription, FilterPipeline, }; use crate::filters::compress_chunk; /// Round a file offset up to the next cache-line boundary. @@ -44,7 +44,8 @@ pub struct ChunkOptions { pub lz4: bool, /// Zstandard compression level (1-22), None = no zstd. Filter ID 32015. pub zstd_level: Option, - /// Pcodec lossless numerical compression. Filter ID 32023. + /// Pcodec lossless numerical compression. Private, unregistered filter + /// ID [`FILTER_PCODEC`] (480): only clawhdf5 can read it. pub pcodec: bool, } @@ -115,7 +116,7 @@ impl ChunkOptions { if self.pcodec { filters.push(FilterDescription { filter_id: FILTER_PCODEC, - name: Some("pcodec".into()), + name: Some(FILTER_PCODEC_NAME.into()), flags: 0, client_data: vec![element_size], }); diff --git a/crates/clawhdf5-format/src/filter_pipeline.rs b/crates/clawhdf5-format/src/filter_pipeline.rs index f64219d..74bc727 100644 --- a/crates/clawhdf5-format/src/filter_pipeline.rs +++ b/crates/clawhdf5-format/src/filter_pipeline.rs @@ -19,8 +19,23 @@ pub const FILTER_SCALEOFFSET: u16 = 6; pub const FILTER_LZ4: u16 = 32004; /// Zstandard compression. pub const FILTER_ZSTD: u16 = 32015; -/// Pcodec lossless numerical codec (clawhdf5 internal; not yet HDF5-registered). -pub const FILTER_PCODEC: u16 = 32023; +/// Pcodec lossless numerical codec — a **private, unregistered** clawhdf5 +/// filter. Pcodec has no ID in the HDF Group's filter registry (checked +/// 2026-09-25, `hdf5_plugins/docs/RegisteredFilterPlugins.md`), so it uses an +/// ID from the registry's testing/private range (256–511). No libhdf5 plugin +/// decodes it: h5py/libhdf5 report the filter as unavailable. Only clawhdf5 +/// (with the `pcodec` feature) reads these datasets. +pub const FILTER_PCODEC: u16 = 480; +/// Filter name written with [`FILTER_PCODEC`]. +pub const FILTER_PCODEC_NAME: &str = "pcodec (clawhdf5 private)"; +/// The ID clawhdf5 up to 2.7.0 wrote pcodec under. It is registered to +/// Granular BitRound (GBR), whose decode is a pass-through, so libhdf5 with +/// that plugin would have returned the compressed bytes as data. Read as +/// pcodec only when the filter is named exactly [`FILTER_PCODEC_LEGACY_NAME`], +/// the name those versions wrote; never written. +pub const FILTER_PCODEC_LEGACY: u16 = 32023; +/// The filter name clawhdf5 up to 2.7.0 wrote with [`FILTER_PCODEC_LEGACY`]. +pub const FILTER_PCODEC_LEGACY_NAME: &str = "pcodec"; /// Description of a single filter in a pipeline. #[derive(Debug, Clone, PartialEq)] diff --git a/crates/clawhdf5-format/src/filters.rs b/crates/clawhdf5-format/src/filters.rs index a28d4be..c0bf701 100644 --- a/crates/clawhdf5-format/src/filters.rs +++ b/crates/clawhdf5-format/src/filters.rs @@ -8,8 +8,9 @@ use alloc::{boxed::Box, vec, vec::Vec}; use crate::error::FormatError; use crate::filter_pipeline::{ - FILTER_DEFLATE, FILTER_FLETCHER32, FILTER_LZ4, FILTER_NBIT, FILTER_PCODEC, FILTER_SCALEOFFSET, - FILTER_SHUFFLE, FILTER_SZIP, FILTER_ZSTD, FilterPipeline, + FILTER_DEFLATE, FILTER_FLETCHER32, FILTER_LZ4, FILTER_NBIT, FILTER_PCODEC, + FILTER_PCODEC_LEGACY, FILTER_PCODEC_LEGACY_NAME, FILTER_SCALEOFFSET, FILTER_SHUFFLE, + FILTER_SZIP, FILTER_ZSTD, FilterPipeline, }; /// Absolute ceiling on a single decompressed chunk's output size, used only @@ -39,6 +40,11 @@ pub fn decompress_chunk( FILTER_ZSTD => zstd_decompress(&data, chunk_size)?, FILTER_FLETCHER32 => fletcher32_verify(&data)?, FILTER_PCODEC => pcodec_decompress(&data, element_size as usize, chunk_size)?, + // Pcodec chunks written by clawhdf5 <= 2.7.0 under the ID registered + // to Granular BitRound; recognised by the name those versions wrote. + FILTER_PCODEC_LEGACY if filter.name.as_deref() == Some(FILTER_PCODEC_LEGACY_NAME) => { + pcodec_decompress(&data, element_size as usize, chunk_size)? + } // `chunk_size` is the expected decompressed size; pass it so these // decoders can reject an element count that would over-allocate. FILTER_SCALEOFFSET => scaleoffset_decompress(&data, &filter.client_data, chunk_size)?, @@ -2192,6 +2198,57 @@ mod tests { assert!(pcodec_decompress(&compressed, 4, 16).is_err()); } + /// Pcodec is written under the private ID 480, not 32023 (registered to + /// Granular BitRound, whose pass-through decode would hand libhdf5 users + /// the compressed bytes as data). Chunks under 32023 are read as pcodec + /// only with the name clawhdf5 <= 2.7.0 wrote. + #[test] + #[cfg(feature = "pcodec")] + fn pcodec_uses_private_id_and_reads_legacy_32023() { + use crate::chunked_write::ChunkOptions; + let opts = ChunkOptions { + pcodec: true, + ..Default::default() + }; + let pl = opts.build_pipeline(8).unwrap(); + let f = pl.filters.iter().find(|f| f.filter_id == 480).unwrap(); + assert_eq!( + f.name.as_deref(), + Some(crate::filter_pipeline::FILTER_PCODEC_NAME) + ); + assert!(pl.filters.iter().all(|f| f.filter_id != 32023)); + + let data: Vec = (0..100).map(|i| i as f64 * 0.25).collect(); + let raw: Vec = data.iter().flat_map(|x| x.to_le_bytes()).collect(); + let compressed = pcodec_compress(&raw, 8).unwrap(); + let pipeline = |id: u16, name: Option<&str>| FilterPipeline { + version: 2, + filters: vec![FilterDescription { + filter_id: id, + name: name.map(Into::into), + flags: 0, + client_data: vec![8], + }], + }; + let legacy = pipeline(32023, Some("pcodec")); + assert_eq!( + decompress_chunk(&compressed, &legacy, raw.len(), 8).unwrap(), + raw + ); + let current = pipeline(480, None); + assert_eq!( + decompress_chunk(&compressed, ¤t, raw.len(), 8).unwrap(), + raw + ); + // A real Granular BitRound filter is not pcodec. + for name in [None, Some("Granular BitRound")] { + assert!(matches!( + decompress_chunk(&compressed, &pipeline(32023, name), raw.len(), 8), + Err(FormatError::UnsupportedFilter(32023)) + )); + } + } + #[test] #[cfg(feature = "lz4")] fn decompress_chunk_rejects_hostile_lz4_size_via_public_entrypoint() { diff --git a/crates/clawhdf5-format/src/type_builders.rs b/crates/clawhdf5-format/src/type_builders.rs index e698899..7b45d70 100644 --- a/crates/clawhdf5-format/src/type_builders.rs +++ b/crates/clawhdf5-format/src/type_builders.rs @@ -671,7 +671,12 @@ impl DatasetBuilder { self } - /// Enable Pcodec lossless numerical compression (clawhdf5 filter ID 32023). + /// Enable Pcodec lossless numerical compression (private clawhdf5 filter + /// ID 480). + /// + /// **Not interoperable:** pcodec has no registered HDF5 filter ID and no + /// libhdf5 plugin, so h5py and other HDF5 readers cannot read the + /// dataset — only clawhdf5 built with the `pcodec` feature can. /// /// Pcodec achieves 30–94% better compression ratio than Zstd for f32/f64 /// columns at 1–5 GiB/s decompression speed (arXiv:2502.06112). Requires