quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases 1 Wiki Activity

fix(format): stop writing pcodec under Granular BitRound's filter ID

Browse Source 
Pcodec chunks were written as filter 32023, which the HDF Group registry
assigns to Granular BitRound (GBR). Pcodec has no registered ID (checked
2026-09-25 against hdf5_plugins/docs/RegisteredFilterPlugins.md, which
ends at 32033 with no pcodec entry). GBR's decode is a pass-through, so
libhdf5 with that plugin loaded would have returned the compressed bytes
as the dataset's values.

Write pcodec as 480, from the registry's testing/private range (256-511),
named "pcodec (clawhdf5 private)", and document it as non-interoperable:
only clawhdf5 with the `pcodec` feature reads it. Chunks under 32023 are
still read as pcodec when the filter is named exactly "pcodec" (what
clawhdf5 <= 2.7.0 wrote); any other 32023 is UnsupportedFilter.

Test: pcodec_uses_private_id_and_reads_legacy_32023.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-25 21:06:07 -05:00
co-authored by Claude Opus 5.5
parent 183d96ee26
commit 06dda26d85
5 changed files with 88 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+1 -1
View File
@@ -696,7 +696,7 @@ stores keep their setting. Opt out with `float16 = false` or
| `fast-checksum` | no | crc32fast-accelerated checksums |
| `lz4` | no | LZ4 block compression filter (id 32004) |
| `zstd` | no | Zstandard compression filter (id 32015) |
| `pcodec` | no | Pcodec lossless numerical codec (id 32023, via `pco` crate) |
| `pcodec` | no | Pcodec lossless numerical codec (via `pco` crate). Private, unregistered filter id 480: **only clawhdf5 can read these datasets** (h5py/libhdf5 cannot). Files from clawhdf5 <= 2.7.0 used id 32023, which is registered to Granular BitRound; they still read. |
| `system-zlib` | no | System zlib backend for deflate (C) |
| `blake3_hash` | no | BLAKE3 content hashing for provenance |
| `szip` | no | SZIP filter (id 4) via libaec (C, through the internal `libaec-sys` crate) |
crates/clawhdf5-format/src/chunked_write.rs
+5 -4
View File
@@ -11,8 +11,8 @@ use crate::chunk_cache::{CACHE_LINE_SIZE, align_to_cache_line};
use crate::ea_writer;
use crate::error::FormatError;
use crate::filter_pipeline::{
FILTER_DEFLATE, FILTER_FLETCHER32, FILTER_LZ4, FILTER_PCODEC, FILTER_SHUFFLE, FILTER_ZSTD,
FilterDescription, FilterPipeline,
FILTER_DEFLATE, FILTER_FLETCHER32, FILTER_LZ4, FILTER_PCODEC, FILTER_PCODEC_NAME,
FILTER_SHUFFLE, FILTER_ZSTD, FilterDescription, FilterPipeline,
};
use crate::filters::compress_chunk;
/// Round a file offset up to the next cache-line boundary.
@@ -44,7 +44,8 @@ pub struct ChunkOptions {
pub lz4: bool,
/// Zstandard compression level (1-22), None = no zstd. Filter ID 32015.
pub zstd_level: Option<u32>,
/// Pcodec lossless numerical compression. Filter ID 32023.
/// Pcodec lossless numerical compression. Private, unregistered filter
/// ID [`FILTER_PCODEC`] (480): only clawhdf5 can read it.
pub pcodec: bool,
}
@@ -115,7 +116,7 @@ impl ChunkOptions {
if self.pcodec {
filters.push(FilterDescription {
filter_id: FILTER_PCODEC,
name: Some("pcodec".into()),
name: Some(FILTER_PCODEC_NAME.into()),
flags: 0,
client_data: vec![element_size],
});
crates/clawhdf5-format/src/filter_pipeline.rs
+17 -2
View File
@@ -19,8 +19,23 @@ pub const FILTER_SCALEOFFSET: u16 = 6;
pub const FILTER_LZ4: u16 = 32004;
/// Zstandard compression.
pub const FILTER_ZSTD: u16 = 32015;
/// Pcodec lossless numerical codec (clawhdf5 internal; not yet HDF5-registered).
pub const FILTER_PCODEC: u16 = 32023;
/// Pcodec lossless numerical codec — a **private, unregistered** clawhdf5
/// filter. Pcodec has no ID in the HDF Group's filter registry (checked
/// 2026-09-25, `hdf5_plugins/docs/RegisteredFilterPlugins.md`), so it uses an
/// ID from the registry's testing/private range (256–511). No libhdf5 plugin
/// decodes it: h5py/libhdf5 report the filter as unavailable. Only clawhdf5
/// (with the `pcodec` feature) reads these datasets.
pub const FILTER_PCODEC: u16 = 480;
/// Filter name written with [`FILTER_PCODEC`].
pub const FILTER_PCODEC_NAME: &str = "pcodec (clawhdf5 private)";
/// The ID clawhdf5 up to 2.7.0 wrote pcodec under. It is registered to
/// Granular BitRound (GBR), whose decode is a pass-through, so libhdf5 with
/// that plugin would have returned the compressed bytes as data. Read as
/// pcodec only when the filter is named exactly [`FILTER_PCODEC_LEGACY_NAME`],
/// the name those versions wrote; never written.
pub const FILTER_PCODEC_LEGACY: u16 = 32023;
/// The filter name clawhdf5 up to 2.7.0 wrote with [`FILTER_PCODEC_LEGACY`].
pub const FILTER_PCODEC_LEGACY_NAME: &str = "pcodec";
/// Description of a single filter in a pipeline.
#[derive(Debug, Clone, PartialEq)]
crates/clawhdf5-format/src/filters.rs
+59 -2
View File
@@ -8,8 +8,9 @@ use alloc::{boxed::Box, vec, vec::Vec};
use crate::error::FormatError;
use crate::filter_pipeline::{
FILTER_DEFLATE, FILTER_FLETCHER32, FILTER_LZ4, FILTER_NBIT, FILTER_PCODEC, FILTER_SCALEOFFSET,
FILTER_SHUFFLE, FILTER_SZIP, FILTER_ZSTD, FilterPipeline,
FILTER_DEFLATE, FILTER_FLETCHER32, FILTER_LZ4, FILTER_NBIT, FILTER_PCODEC,
FILTER_PCODEC_LEGACY, FILTER_PCODEC_LEGACY_NAME, FILTER_SCALEOFFSET, FILTER_SHUFFLE,
FILTER_SZIP, FILTER_ZSTD, FilterPipeline,
};
/// Absolute ceiling on a single decompressed chunk's output size, used only
@@ -39,6 +40,11 @@ pub fn decompress_chunk(
FILTER_ZSTD => zstd_decompress(&data, chunk_size)?,
FILTER_FLETCHER32 => fletcher32_verify(&data)?,
FILTER_PCODEC => pcodec_decompress(&data, element_size as usize, chunk_size)?,
// Pcodec chunks written by clawhdf5 <= 2.7.0 under the ID registered
// to Granular BitRound; recognised by the name those versions wrote.
FILTER_PCODEC_LEGACY if filter.name.as_deref() == Some(FILTER_PCODEC_LEGACY_NAME) => {
pcodec_decompress(&data, element_size as usize, chunk_size)?
}
// `chunk_size` is the expected decompressed size; pass it so these
// decoders can reject an element count that would over-allocate.
FILTER_SCALEOFFSET => scaleoffset_decompress(&data, &filter.client_data, chunk_size)?,
@@ -2192,6 +2198,57 @@ mod tests {
assert!(pcodec_decompress(&compressed, 4, 16).is_err());
}
/// Pcodec is written under the private ID 480, not 32023 (registered to
/// Granular BitRound, whose pass-through decode would hand libhdf5 users
/// the compressed bytes as data). Chunks under 32023 are read as pcodec
/// only with the name clawhdf5 <= 2.7.0 wrote.
#[test]
#[cfg(feature = "pcodec")]
fn pcodec_uses_private_id_and_reads_legacy_32023() {
use crate::chunked_write::ChunkOptions;
let opts = ChunkOptions {
pcodec: true,
..Default::default()
};
let pl = opts.build_pipeline(8).unwrap();
let f = pl.filters.iter().find(|f| f.filter_id == 480).unwrap();
assert_eq!(
f.name.as_deref(),
Some(crate::filter_pipeline::FILTER_PCODEC_NAME)
);
assert!(pl.filters.iter().all(|f| f.filter_id != 32023));
let data: Vec<f64> = (0..100).map(|i| i as f64 * 0.25).collect();
let raw: Vec<u8> = data.iter().flat_map(|x| x.to_le_bytes()).collect();
let compressed = pcodec_compress(&raw, 8).unwrap();
let pipeline = |id: u16, name: Option<&str>| FilterPipeline {
version: 2,
filters: vec![FilterDescription {
filter_id: id,
name: name.map(Into::into),
flags: 0,
client_data: vec![8],
}],
};
let legacy = pipeline(32023, Some("pcodec"));
assert_eq!(
decompress_chunk(&compressed, &legacy, raw.len(), 8).unwrap(),
raw
);
let current = pipeline(480, None);
assert_eq!(
decompress_chunk(&compressed, &current, raw.len(), 8).unwrap(),
raw
);
// A real Granular BitRound filter is not pcodec.
for name in [None, Some("Granular BitRound")] {
assert!(matches!(
decompress_chunk(&compressed, &pipeline(32023, name), raw.len(), 8),
Err(FormatError::UnsupportedFilter(32023))
));
}
}
#[test]
#[cfg(feature = "lz4")]
fn decompress_chunk_rejects_hostile_lz4_size_via_public_entrypoint() {
crates/clawhdf5-format/src/type_builders.rs
+6 -1
View File
@@ -671,7 +671,12 @@ impl DatasetBuilder {
self
}
/// Enable Pcodec lossless numerical compression (clawhdf5 filter ID 32023).
/// Enable Pcodec lossless numerical compression (private clawhdf5 filter
/// ID 480).
///
/// **Not interoperable:** pcodec has no registered HDF5 filter ID and no
/// libhdf5 plugin, so h5py and other HDF5 readers cannot read the
/// dataset — only clawhdf5 built with the `pcodec` feature can.
///
/// Pcodec achieves 30–94% better compression ratio than Zstd for f32/f64
/// columns at 1–5 GiB/s decompression speed (arXiv:2502.06112). Requires