From f37e7ae3263277319dba4bc39be5397194eb00c3 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 19:33:03 -0500 Subject: [PATCH] format: bound ZFP chunk fetches at 4x, not n + n/4 The M2 fetch bound gave every codec n + n/4 + 4096 stored bytes, but ZFP's fixed-rate mode stores up to 64 bits per value, doubling 4-byte types: 60 of the 2205 zfp_interop datasets (rate 64, f4/i4) failed with 'compressed stream ends early' once M2 and ZFP were merged. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/filters.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/crates/clawhdf5-format/src/filters.rs b/crates/clawhdf5-format/src/filters.rs index 98cf270..28070d4 100644 --- a/crates/clawhdf5-format/src/filters.rs +++ b/crates/clawhdf5-format/src/filters.rs @@ -72,7 +72,8 @@ fn filter_output_bound(filter_id: u16, input: usize) -> usize { /// `filter_mask`) is the chunk itself: `chunk_bytes`, and bytes past them /// were never used. A filtered chunk is bounded by each applied filter's /// worst-case growth in the write direction: shuffle keeps the size, -/// Fletcher32 adds 4 bytes, and any other codec gets `n + n/4 + 4096` — a +/// Fletcher32 adds 4 bytes, ZFP gets `4n + 4096` (its fixed-rate mode stores +/// up to 64 bits per value), and any other codec gets `n + n/4 + 4096` — a /// deliberately generous bound (bzip2 grows 1000 random bytes by 252, more /// than the decoders' own `n + n/8 + 64` output bound), since a legitimate /// chunk cut short here would fail to read. A filter handled by a codec the @@ -95,6 +96,11 @@ pub(crate) fn stored_chunk_limit( FILTER_SHUFFLE => size, FILTER_FLETCHER32 => size.saturating_add(4), id if filter_registry::may_be_registered(id) => return usize::MAX, + // ZFP's fixed-rate mode stores up to 64 bits per value, so a + // 4-byte type doubles, and precision/accuracy modes add group + // test bits per bit plane on top: 4x plus a header is still a + // bound, where `n + n/4` cut rate-64 chunks short. + crate::filter_pipeline::FILTER_ZFP => size.saturating_mul(4).saturating_add(4096), _ => size.saturating_add(size / 4).saturating_add(4096), }; }