Fix silent wrong data and libhdf5 interop found by the HDF5 audit #11
@@ -696,7 +696,7 @@ stores keep their setting. Opt out with `float16 = false` or
|
|||||||
| `fast-checksum` | no | crc32fast-accelerated checksums |
|
| `fast-checksum` | no | crc32fast-accelerated checksums |
|
||||||
| `lz4` | no | LZ4 block compression filter (id 32004) |
|
| `lz4` | no | LZ4 block compression filter (id 32004) |
|
||||||
| `zstd` | no | Zstandard compression filter (id 32015) |
|
| `zstd` | no | Zstandard compression filter (id 32015) |
|
||||||
| `pcodec` | no | Pcodec lossless numerical codec (id 32023, via `pco` crate) |
|
| `pcodec` | no | Pcodec lossless numerical codec (via `pco` crate). Private, unregistered filter id 480: **only clawhdf5 can read these datasets** (h5py/libhdf5 cannot). Files from clawhdf5 <= 2.7.0 used id 32023, which is registered to Granular BitRound; they still read. |
|
||||||
| `system-zlib` | no | System zlib backend for deflate (C) |
|
| `system-zlib` | no | System zlib backend for deflate (C) |
|
||||||
| `blake3_hash` | no | BLAKE3 content hashing for provenance |
|
| `blake3_hash` | no | BLAKE3 content hashing for provenance |
|
||||||
| `szip` | no | SZIP filter (id 4) via libaec (C, through the internal `libaec-sys` crate) |
|
| `szip` | no | SZIP filter (id 4) via libaec (C, through the internal `libaec-sys` crate) |
|
||||||
|
|||||||
@@ -11,8 +11,8 @@ use crate::chunk_cache::{CACHE_LINE_SIZE, align_to_cache_line};
|
|||||||
use crate::ea_writer;
|
use crate::ea_writer;
|
||||||
use crate::error::FormatError;
|
use crate::error::FormatError;
|
||||||
use crate::filter_pipeline::{
|
use crate::filter_pipeline::{
|
||||||
FILTER_DEFLATE, FILTER_FLETCHER32, FILTER_LZ4, FILTER_PCODEC, FILTER_SHUFFLE, FILTER_ZSTD,
|
FILTER_DEFLATE, FILTER_FLETCHER32, FILTER_LZ4, FILTER_PCODEC, FILTER_PCODEC_NAME,
|
||||||
FilterDescription, FilterPipeline,
|
FILTER_SHUFFLE, FILTER_ZSTD, FilterDescription, FilterPipeline,
|
||||||
};
|
};
|
||||||
use crate::filters::compress_chunk;
|
use crate::filters::compress_chunk;
|
||||||
/// Round a file offset up to the next cache-line boundary.
|
/// Round a file offset up to the next cache-line boundary.
|
||||||
@@ -44,7 +44,8 @@ pub struct ChunkOptions {
|
|||||||
pub lz4: bool,
|
pub lz4: bool,
|
||||||
/// Zstandard compression level (1-22), None = no zstd. Filter ID 32015.
|
/// Zstandard compression level (1-22), None = no zstd. Filter ID 32015.
|
||||||
pub zstd_level: Option<u32>,
|
pub zstd_level: Option<u32>,
|
||||||
/// Pcodec lossless numerical compression. Filter ID 32023.
|
/// Pcodec lossless numerical compression. Private, unregistered filter
|
||||||
|
/// ID [`FILTER_PCODEC`] (480): only clawhdf5 can read it.
|
||||||
pub pcodec: bool,
|
pub pcodec: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -115,7 +116,7 @@ impl ChunkOptions {
|
|||||||
if self.pcodec {
|
if self.pcodec {
|
||||||
filters.push(FilterDescription {
|
filters.push(FilterDescription {
|
||||||
filter_id: FILTER_PCODEC,
|
filter_id: FILTER_PCODEC,
|
||||||
name: Some("pcodec".into()),
|
name: Some(FILTER_PCODEC_NAME.into()),
|
||||||
flags: 0,
|
flags: 0,
|
||||||
client_data: vec![element_size],
|
client_data: vec![element_size],
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -19,8 +19,23 @@ pub const FILTER_SCALEOFFSET: u16 = 6;
|
|||||||
pub const FILTER_LZ4: u16 = 32004;
|
pub const FILTER_LZ4: u16 = 32004;
|
||||||
/// Zstandard compression.
|
/// Zstandard compression.
|
||||||
pub const FILTER_ZSTD: u16 = 32015;
|
pub const FILTER_ZSTD: u16 = 32015;
|
||||||
/// Pcodec lossless numerical codec (clawhdf5 internal; not yet HDF5-registered).
|
/// Pcodec lossless numerical codec — a **private, unregistered** clawhdf5
|
||||||
pub const FILTER_PCODEC: u16 = 32023;
|
/// filter. Pcodec has no ID in the HDF Group's filter registry (checked
|
||||||
|
/// 2026-09-25, `hdf5_plugins/docs/RegisteredFilterPlugins.md`), so it uses an
|
||||||
|
/// ID from the registry's testing/private range (256–511). No libhdf5 plugin
|
||||||
|
/// decodes it: h5py/libhdf5 report the filter as unavailable. Only clawhdf5
|
||||||
|
/// (with the `pcodec` feature) reads these datasets.
|
||||||
|
pub const FILTER_PCODEC: u16 = 480;
|
||||||
|
/// Filter name written with [`FILTER_PCODEC`].
|
||||||
|
pub const FILTER_PCODEC_NAME: &str = "pcodec (clawhdf5 private)";
|
||||||
|
/// The ID clawhdf5 up to 2.7.0 wrote pcodec under. It is registered to
|
||||||
|
/// Granular BitRound (GBR), whose decode is a pass-through, so libhdf5 with
|
||||||
|
/// that plugin would have returned the compressed bytes as data. Read as
|
||||||
|
/// pcodec only when the filter is named exactly [`FILTER_PCODEC_LEGACY_NAME`],
|
||||||
|
/// the name those versions wrote; never written.
|
||||||
|
pub const FILTER_PCODEC_LEGACY: u16 = 32023;
|
||||||
|
/// The filter name clawhdf5 up to 2.7.0 wrote with [`FILTER_PCODEC_LEGACY`].
|
||||||
|
pub const FILTER_PCODEC_LEGACY_NAME: &str = "pcodec";
|
||||||
|
|
||||||
/// Description of a single filter in a pipeline.
|
/// Description of a single filter in a pipeline.
|
||||||
#[derive(Debug, Clone, PartialEq)]
|
#[derive(Debug, Clone, PartialEq)]
|
||||||
|
|||||||
@@ -8,8 +8,9 @@ use alloc::{boxed::Box, vec, vec::Vec};
|
|||||||
|
|
||||||
use crate::error::FormatError;
|
use crate::error::FormatError;
|
||||||
use crate::filter_pipeline::{
|
use crate::filter_pipeline::{
|
||||||
FILTER_DEFLATE, FILTER_FLETCHER32, FILTER_LZ4, FILTER_NBIT, FILTER_PCODEC, FILTER_SCALEOFFSET,
|
FILTER_DEFLATE, FILTER_FLETCHER32, FILTER_LZ4, FILTER_NBIT, FILTER_PCODEC,
|
||||||
FILTER_SHUFFLE, FILTER_SZIP, FILTER_ZSTD, FilterPipeline,
|
FILTER_PCODEC_LEGACY, FILTER_PCODEC_LEGACY_NAME, FILTER_SCALEOFFSET, FILTER_SHUFFLE,
|
||||||
|
FILTER_SZIP, FILTER_ZSTD, FilterPipeline,
|
||||||
};
|
};
|
||||||
|
|
||||||
/// Absolute ceiling on a single decompressed chunk's output size, used only
|
/// Absolute ceiling on a single decompressed chunk's output size, used only
|
||||||
@@ -39,6 +40,11 @@ pub fn decompress_chunk(
|
|||||||
FILTER_ZSTD => zstd_decompress(&data, chunk_size)?,
|
FILTER_ZSTD => zstd_decompress(&data, chunk_size)?,
|
||||||
FILTER_FLETCHER32 => fletcher32_verify(&data)?,
|
FILTER_FLETCHER32 => fletcher32_verify(&data)?,
|
||||||
FILTER_PCODEC => pcodec_decompress(&data, element_size as usize, chunk_size)?,
|
FILTER_PCODEC => pcodec_decompress(&data, element_size as usize, chunk_size)?,
|
||||||
|
// Pcodec chunks written by clawhdf5 <= 2.7.0 under the ID registered
|
||||||
|
// to Granular BitRound; recognised by the name those versions wrote.
|
||||||
|
FILTER_PCODEC_LEGACY if filter.name.as_deref() == Some(FILTER_PCODEC_LEGACY_NAME) => {
|
||||||
|
pcodec_decompress(&data, element_size as usize, chunk_size)?
|
||||||
|
}
|
||||||
// `chunk_size` is the expected decompressed size; pass it so these
|
// `chunk_size` is the expected decompressed size; pass it so these
|
||||||
// decoders can reject an element count that would over-allocate.
|
// decoders can reject an element count that would over-allocate.
|
||||||
FILTER_SCALEOFFSET => scaleoffset_decompress(&data, &filter.client_data, chunk_size)?,
|
FILTER_SCALEOFFSET => scaleoffset_decompress(&data, &filter.client_data, chunk_size)?,
|
||||||
@@ -2192,6 +2198,57 @@ mod tests {
|
|||||||
assert!(pcodec_decompress(&compressed, 4, 16).is_err());
|
assert!(pcodec_decompress(&compressed, 4, 16).is_err());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Pcodec is written under the private ID 480, not 32023 (registered to
|
||||||
|
/// Granular BitRound, whose pass-through decode would hand libhdf5 users
|
||||||
|
/// the compressed bytes as data). Chunks under 32023 are read as pcodec
|
||||||
|
/// only with the name clawhdf5 <= 2.7.0 wrote.
|
||||||
|
#[test]
|
||||||
|
#[cfg(feature = "pcodec")]
|
||||||
|
fn pcodec_uses_private_id_and_reads_legacy_32023() {
|
||||||
|
use crate::chunked_write::ChunkOptions;
|
||||||
|
let opts = ChunkOptions {
|
||||||
|
pcodec: true,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let pl = opts.build_pipeline(8).unwrap();
|
||||||
|
let f = pl.filters.iter().find(|f| f.filter_id == 480).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
f.name.as_deref(),
|
||||||
|
Some(crate::filter_pipeline::FILTER_PCODEC_NAME)
|
||||||
|
);
|
||||||
|
assert!(pl.filters.iter().all(|f| f.filter_id != 32023));
|
||||||
|
|
||||||
|
let data: Vec<f64> = (0..100).map(|i| i as f64 * 0.25).collect();
|
||||||
|
let raw: Vec<u8> = data.iter().flat_map(|x| x.to_le_bytes()).collect();
|
||||||
|
let compressed = pcodec_compress(&raw, 8).unwrap();
|
||||||
|
let pipeline = |id: u16, name: Option<&str>| FilterPipeline {
|
||||||
|
version: 2,
|
||||||
|
filters: vec![FilterDescription {
|
||||||
|
filter_id: id,
|
||||||
|
name: name.map(Into::into),
|
||||||
|
flags: 0,
|
||||||
|
client_data: vec![8],
|
||||||
|
}],
|
||||||
|
};
|
||||||
|
let legacy = pipeline(32023, Some("pcodec"));
|
||||||
|
assert_eq!(
|
||||||
|
decompress_chunk(&compressed, &legacy, raw.len(), 8).unwrap(),
|
||||||
|
raw
|
||||||
|
);
|
||||||
|
let current = pipeline(480, None);
|
||||||
|
assert_eq!(
|
||||||
|
decompress_chunk(&compressed, ¤t, raw.len(), 8).unwrap(),
|
||||||
|
raw
|
||||||
|
);
|
||||||
|
// A real Granular BitRound filter is not pcodec.
|
||||||
|
for name in [None, Some("Granular BitRound")] {
|
||||||
|
assert!(matches!(
|
||||||
|
decompress_chunk(&compressed, &pipeline(32023, name), raw.len(), 8),
|
||||||
|
Err(FormatError::UnsupportedFilter(32023))
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[cfg(feature = "lz4")]
|
#[cfg(feature = "lz4")]
|
||||||
fn decompress_chunk_rejects_hostile_lz4_size_via_public_entrypoint() {
|
fn decompress_chunk_rejects_hostile_lz4_size_via_public_entrypoint() {
|
||||||
|
|||||||
@@ -671,7 +671,12 @@ impl DatasetBuilder {
|
|||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Enable Pcodec lossless numerical compression (clawhdf5 filter ID 32023).
|
/// Enable Pcodec lossless numerical compression (private clawhdf5 filter
|
||||||
|
/// ID 480).
|
||||||
|
///
|
||||||
|
/// **Not interoperable:** pcodec has no registered HDF5 filter ID and no
|
||||||
|
/// libhdf5 plugin, so h5py and other HDF5 readers cannot read the
|
||||||
|
/// dataset — only clawhdf5 built with the `pcodec` feature can.
|
||||||
///
|
///
|
||||||
/// Pcodec achieves 30–94% better compression ratio than Zstd for f32/f64
|
/// Pcodec achieves 30–94% better compression ratio than Zstd for f32/f64
|
||||||
/// columns at 1–5 GiB/s decompression speed (arXiv:2502.06112). Requires
|
/// columns at 1–5 GiB/s decompression speed (arXiv:2502.06112). Requires
|
||||||
|
|||||||
Reference in New Issue
Block a user