Fix silent wrong data and libhdf5 interop found by the HDF5 audit #11

Merged
osobh merged 41 commits from fix/phase0-correctness into main 2026-09-26 02:42:54 +00:00
5 changed files with 88 additions and 10 deletions
Showing only changes of commit 06dda26d85 - Show all commits
+1 -1
View File
@@ -696,7 +696,7 @@ stores keep their setting. Opt out with `float16 = false` or
| `fast-checksum` | no | crc32fast-accelerated checksums |
| `lz4` | no | LZ4 block compression filter (id 32004) |
| `zstd` | no | Zstandard compression filter (id 32015) |
| `pcodec` | no | Pcodec lossless numerical codec (id 32023, via `pco` crate) |
| `pcodec` | no | Pcodec lossless numerical codec (via `pco` crate). Private, unregistered filter id 480: **only clawhdf5 can read these datasets** (h5py/libhdf5 cannot). Files from clawhdf5 <= 2.7.0 used id 32023, which is registered to Granular BitRound; they still read. |
| `system-zlib` | no | System zlib backend for deflate (C) |
| `blake3_hash` | no | BLAKE3 content hashing for provenance |
| `szip` | no | SZIP filter (id 4) via libaec (C, through the internal `libaec-sys` crate) |
+5 -4
View File
@@ -11,8 +11,8 @@ use crate::chunk_cache::{CACHE_LINE_SIZE, align_to_cache_line};
use crate::ea_writer;
use crate::error::FormatError;
use crate::filter_pipeline::{
FILTER_DEFLATE, FILTER_FLETCHER32, FILTER_LZ4, FILTER_PCODEC, FILTER_SHUFFLE, FILTER_ZSTD,
FilterDescription, FilterPipeline,
FILTER_DEFLATE, FILTER_FLETCHER32, FILTER_LZ4, FILTER_PCODEC, FILTER_PCODEC_NAME,
FILTER_SHUFFLE, FILTER_ZSTD, FilterDescription, FilterPipeline,
};
use crate::filters::compress_chunk;
/// Round a file offset up to the next cache-line boundary.
@@ -44,7 +44,8 @@ pub struct ChunkOptions {
pub lz4: bool,
/// Zstandard compression level (1-22), None = no zstd. Filter ID 32015.
pub zstd_level: Option<u32>,
/// Pcodec lossless numerical compression. Filter ID 32023.
/// Pcodec lossless numerical compression. Private, unregistered filter
/// ID [`FILTER_PCODEC`] (480): only clawhdf5 can read it.
pub pcodec: bool,
}
@@ -115,7 +116,7 @@ impl ChunkOptions {
if self.pcodec {
filters.push(FilterDescription {
filter_id: FILTER_PCODEC,
name: Some("pcodec".into()),
name: Some(FILTER_PCODEC_NAME.into()),
flags: 0,
client_data: vec![element_size],
});
+17 -2
View File
@@ -19,8 +19,23 @@ pub const FILTER_SCALEOFFSET: u16 = 6;
pub const FILTER_LZ4: u16 = 32004;
/// Zstandard compression.
pub const FILTER_ZSTD: u16 = 32015;
/// Pcodec lossless numerical codec (clawhdf5 internal; not yet HDF5-registered).
pub const FILTER_PCODEC: u16 = 32023;
/// Pcodec lossless numerical codec — a **private, unregistered** clawhdf5
/// filter. Pcodec has no ID in the HDF Group's filter registry (checked
/// 2026-09-25, `hdf5_plugins/docs/RegisteredFilterPlugins.md`), so it uses an
/// ID from the registry's testing/private range (256–511). No libhdf5 plugin
/// decodes it: h5py/libhdf5 report the filter as unavailable. Only clawhdf5
/// (with the `pcodec` feature) reads these datasets.
pub const FILTER_PCODEC: u16 = 480;
/// Filter name written with [`FILTER_PCODEC`].
pub const FILTER_PCODEC_NAME: &str = "pcodec (clawhdf5 private)";
/// The ID clawhdf5 up to 2.7.0 wrote pcodec under. It is registered to
/// Granular BitRound (GBR), whose decode is a pass-through, so libhdf5 with
/// that plugin would have returned the compressed bytes as data. Read as
/// pcodec only when the filter is named exactly [`FILTER_PCODEC_LEGACY_NAME`],
/// the name those versions wrote; never written.
pub const FILTER_PCODEC_LEGACY: u16 = 32023;
/// The filter name clawhdf5 up to 2.7.0 wrote with [`FILTER_PCODEC_LEGACY`].
pub const FILTER_PCODEC_LEGACY_NAME: &str = "pcodec";
/// Description of a single filter in a pipeline.
#[derive(Debug, Clone, PartialEq)]
+59 -2
View File
@@ -8,8 +8,9 @@ use alloc::{boxed::Box, vec, vec::Vec};
use crate::error::FormatError;
use crate::filter_pipeline::{
FILTER_DEFLATE, FILTER_FLETCHER32, FILTER_LZ4, FILTER_NBIT, FILTER_PCODEC, FILTER_SCALEOFFSET,
FILTER_SHUFFLE, FILTER_SZIP, FILTER_ZSTD, FilterPipeline,
FILTER_DEFLATE, FILTER_FLETCHER32, FILTER_LZ4, FILTER_NBIT, FILTER_PCODEC,
FILTER_PCODEC_LEGACY, FILTER_PCODEC_LEGACY_NAME, FILTER_SCALEOFFSET, FILTER_SHUFFLE,
FILTER_SZIP, FILTER_ZSTD, FilterPipeline,
};
/// Absolute ceiling on a single decompressed chunk's output size, used only
@@ -39,6 +40,11 @@ pub fn decompress_chunk(
FILTER_ZSTD => zstd_decompress(&data, chunk_size)?,
FILTER_FLETCHER32 => fletcher32_verify(&data)?,
FILTER_PCODEC => pcodec_decompress(&data, element_size as usize, chunk_size)?,
// Pcodec chunks written by clawhdf5 <= 2.7.0 under the ID registered
// to Granular BitRound; recognised by the name those versions wrote.
FILTER_PCODEC_LEGACY if filter.name.as_deref() == Some(FILTER_PCODEC_LEGACY_NAME) => {
pcodec_decompress(&data, element_size as usize, chunk_size)?
}
// `chunk_size` is the expected decompressed size; pass it so these
// decoders can reject an element count that would over-allocate.
FILTER_SCALEOFFSET => scaleoffset_decompress(&data, &filter.client_data, chunk_size)?,
@@ -2192,6 +2198,57 @@ mod tests {
assert!(pcodec_decompress(&compressed, 4, 16).is_err());
}
/// Pcodec is written under the private ID 480, not 32023 (registered to
/// Granular BitRound, whose pass-through decode would hand libhdf5 users
/// the compressed bytes as data). Chunks under 32023 are read as pcodec
/// only with the name clawhdf5 <= 2.7.0 wrote.
#[test]
#[cfg(feature = "pcodec")]
fn pcodec_uses_private_id_and_reads_legacy_32023() {
use crate::chunked_write::ChunkOptions;
let opts = ChunkOptions {
pcodec: true,
..Default::default()
};
let pl = opts.build_pipeline(8).unwrap();
let f = pl.filters.iter().find(|f| f.filter_id == 480).unwrap();
assert_eq!(
f.name.as_deref(),
Some(crate::filter_pipeline::FILTER_PCODEC_NAME)
);
assert!(pl.filters.iter().all(|f| f.filter_id != 32023));
let data: Vec<f64> = (0..100).map(|i| i as f64 * 0.25).collect();
let raw: Vec<u8> = data.iter().flat_map(|x| x.to_le_bytes()).collect();
let compressed = pcodec_compress(&raw, 8).unwrap();
let pipeline = |id: u16, name: Option<&str>| FilterPipeline {
version: 2,
filters: vec![FilterDescription {
filter_id: id,
name: name.map(Into::into),
flags: 0,
client_data: vec![8],
}],
};
let legacy = pipeline(32023, Some("pcodec"));
assert_eq!(
decompress_chunk(&compressed, &legacy, raw.len(), 8).unwrap(),
raw
);
let current = pipeline(480, None);
assert_eq!(
decompress_chunk(&compressed, &current, raw.len(), 8).unwrap(),
raw
);
// A real Granular BitRound filter is not pcodec.
for name in [None, Some("Granular BitRound")] {
assert!(matches!(
decompress_chunk(&compressed, &pipeline(32023, name), raw.len(), 8),
Err(FormatError::UnsupportedFilter(32023))
));
}
}
#[test]
#[cfg(feature = "lz4")]
fn decompress_chunk_rejects_hostile_lz4_size_via_public_entrypoint() {
+6 -1
View File
@@ -671,7 +671,12 @@ impl DatasetBuilder {
self
}
/// Enable Pcodec lossless numerical compression (clawhdf5 filter ID 32023).
/// Enable Pcodec lossless numerical compression (private clawhdf5 filter
/// ID 480).
///
/// **Not interoperable:** pcodec has no registered HDF5 filter ID and no
/// libhdf5 plugin, so h5py and other HDF5 readers cannot read the
/// dataset — only clawhdf5 built with the `pcodec` feature can.
///
/// Pcodec achieves 30–94% better compression ratio than Zstd for f32/f64
/// columns at 1–5 GiB/s decompression speed (arXiv:2502.06112). Requires