fix(format): bound each filter stage by what the stages before it produce

Every decode stage was capped at the chunk's decoded size. That holds
only when every filter ahead of the codec preserves size; Fletcher32
does not (it appends a 4-byte checksum), so a pipeline with Fletcher32
before deflate (NetCDF-4's fletcher32 -> shuffle -> deflate ordering,
h5repack's "all filters") failed with "deflate: output exceeds size
limit" on every chunk.

decompress_chunk_masked now computes each stage's bound by running the
chunk size forward through the filters that precede it in write order
(and that the chunk's mask did not skip): shuffle keeps the size,
Fletcher32 adds 4, any codec adds at most n/8 + 64. The cap is still a
small constant factor of the chunk, so a decompression bomb is rejected
as before (tested).

Shuffle also had to learn libhdf5's handling of a length that is not a
whole number of elements (chunk + checksum): shuffle the whole elements
and leave the trailing bytes in place, in both directions. It used to
refuse such data.

Regression: h5py_fletcher32_before_deflate_reads (fletcher->shuffle->
gzip, fletcher->gzip, shuffle->fletcher->gzip, and a 2-D i32 grid),
fletcher32_ahead_of_deflate_stays_bounded and
shuffle_leaves_a_partial_trailing_element_in_place.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-25 21:06:22 -05:00
co-authored by Claude Opus 5.5
parent 585e14d5e2
commit e162c013fd
2 changed files with 160 additions and 15 deletions
@@ -173,3 +173,67 @@ with h5py.File("{p}", "w") as f:
let want: Vec<u8> = (1006..1026i32).flat_map(i32::to_le_bytes).collect();
assert_eq!(part, want);
}
// ---------------------------------------------------------------------------
// Size-changing filters ahead of a codec
// ---------------------------------------------------------------------------
/// Fletcher32 placed before the compressor (NetCDF-4's ordering) makes the
/// codec's decoded output 4 bytes larger than the chunk. The decompression
/// cap was the chunk size for every stage, so these files failed with
/// "deflate: output exceeds size limit".
#[test]
fn h5py_fletcher32_before_deflate_reads() {
skip_if_no_python!();
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("fletcher_first.h5");
let p = path.display().to_string();
run_python(&format!(
r#"
import h5py, numpy as np
arr = np.sin(np.arange(5000) / 50.0)
grid = np.arange(37 * 21, dtype="<i4").reshape(37, 21)
ids = {{"fletcher32": 3, "shuffle": 2, "deflate": 1}}
with h5py.File("{p}", "w") as f:
for name, steps, data, chunk in (
("fl_shuf_gzip", ("fletcher32", "shuffle", "deflate"), arr, (500,)),
("fl_gzip", ("fletcher32", "deflate"), arr, (500,)),
("shuf_fl_gzip", ("shuffle", "fletcher32", "deflate"), arr, (500,)),
("grid_fl_shuf_gzip", ("fletcher32", "shuffle", "deflate"), grid, (8, 5)),
):
dcpl = h5py.h5p.create(h5py.h5p.DATASET_CREATE)
dcpl.set_chunk(chunk)
for s in steps:
if s == "deflate":
dcpl.set_deflate(4)
elif s == "shuffle":
dcpl.set_shuffle()
else:
dcpl.set_fletcher32()
tid = h5py.h5t.py_create(data.dtype)
space = h5py.h5s.create_simple(data.shape)
d = h5py.h5d.create(f.id, name.encode(), tid, space, dcpl=dcpl)
d.write(h5py.h5s.ALL, h5py.h5s.ALL, np.ascontiguousarray(data))
order = [d.get_create_plist().get_filter(i)[0] for i in range(len(steps))]
assert order == [ids[s] for s in steps], order
"#
));
let file = File::open(&path).unwrap();
let arr: Vec<f64> = (0..5000).map(|i| (f64::from(i) / 50.0).sin()).collect();
for name in ["fl_shuf_gzip", "fl_gzip", "shuf_fl_gzip"] {
let values = file.dataset(name).unwrap().read_f64().unwrap();
assert_eq!(values.len(), arr.len(), "{name}");
for (i, (v, w)) in values.iter().zip(&arr).enumerate() {
assert!((v - w).abs() < 1e-12, "{name}[{i}]: {v} vs {w}");
}
}
let grid: Vec<i32> = (0..37 * 21).collect();
assert_eq!(
file.dataset("grid_fl_shuf_gzip")
.unwrap()
.read_i32()
.unwrap(),
grid
);
}