fix(format): bound each filter stage by what the stages before it produce
Every decode stage was capped at the chunk's decoded size. That holds only when every filter ahead of the codec preserves size; Fletcher32 does not (it appends a 4-byte checksum), so a pipeline with Fletcher32 before deflate (NetCDF-4's fletcher32 -> shuffle -> deflate ordering, h5repack's "all filters") failed with "deflate: output exceeds size limit" on every chunk. decompress_chunk_masked now computes each stage's bound by running the chunk size forward through the filters that precede it in write order (and that the chunk's mask did not skip): shuffle keeps the size, Fletcher32 adds 4, any codec adds at most n/8 + 64. The cap is still a small constant factor of the chunk, so a decompression bomb is rejected as before (tested). Shuffle also had to learn libhdf5's handling of a length that is not a whole number of elements (chunk + checksum): shuffle the whole elements and leave the trailing bytes in place, in both directions. It used to refuse such data. Regression: h5py_fletcher32_before_deflate_reads (fletcher->shuffle-> gzip, fletcher->gzip, shuffle->fletcher->gzip, and a 2-D i32 grid), fletcher32_ahead_of_deflate_stays_bounded and shuffle_leaves_a_partial_trailing_element_in_place. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
@@ -173,3 +173,67 @@ with h5py.File("{p}", "w") as f:
|
||||
let want: Vec<u8> = (1006..1026i32).flat_map(i32::to_le_bytes).collect();
|
||||
assert_eq!(part, want);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Size-changing filters ahead of a codec
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Fletcher32 placed before the compressor (NetCDF-4's ordering) makes the
|
||||
/// codec's decoded output 4 bytes larger than the chunk. The decompression
|
||||
/// cap was the chunk size for every stage, so these files failed with
|
||||
/// "deflate: output exceeds size limit".
|
||||
#[test]
|
||||
fn h5py_fletcher32_before_deflate_reads() {
|
||||
skip_if_no_python!();
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("fletcher_first.h5");
|
||||
let p = path.display().to_string();
|
||||
run_python(&format!(
|
||||
r#"
|
||||
import h5py, numpy as np
|
||||
arr = np.sin(np.arange(5000) / 50.0)
|
||||
grid = np.arange(37 * 21, dtype="<i4").reshape(37, 21)
|
||||
ids = {{"fletcher32": 3, "shuffle": 2, "deflate": 1}}
|
||||
with h5py.File("{p}", "w") as f:
|
||||
for name, steps, data, chunk in (
|
||||
("fl_shuf_gzip", ("fletcher32", "shuffle", "deflate"), arr, (500,)),
|
||||
("fl_gzip", ("fletcher32", "deflate"), arr, (500,)),
|
||||
("shuf_fl_gzip", ("shuffle", "fletcher32", "deflate"), arr, (500,)),
|
||||
("grid_fl_shuf_gzip", ("fletcher32", "shuffle", "deflate"), grid, (8, 5)),
|
||||
):
|
||||
dcpl = h5py.h5p.create(h5py.h5p.DATASET_CREATE)
|
||||
dcpl.set_chunk(chunk)
|
||||
for s in steps:
|
||||
if s == "deflate":
|
||||
dcpl.set_deflate(4)
|
||||
elif s == "shuffle":
|
||||
dcpl.set_shuffle()
|
||||
else:
|
||||
dcpl.set_fletcher32()
|
||||
tid = h5py.h5t.py_create(data.dtype)
|
||||
space = h5py.h5s.create_simple(data.shape)
|
||||
d = h5py.h5d.create(f.id, name.encode(), tid, space, dcpl=dcpl)
|
||||
d.write(h5py.h5s.ALL, h5py.h5s.ALL, np.ascontiguousarray(data))
|
||||
order = [d.get_create_plist().get_filter(i)[0] for i in range(len(steps))]
|
||||
assert order == [ids[s] for s in steps], order
|
||||
"#
|
||||
));
|
||||
|
||||
let file = File::open(&path).unwrap();
|
||||
let arr: Vec<f64> = (0..5000).map(|i| (f64::from(i) / 50.0).sin()).collect();
|
||||
for name in ["fl_shuf_gzip", "fl_gzip", "shuf_fl_gzip"] {
|
||||
let values = file.dataset(name).unwrap().read_f64().unwrap();
|
||||
assert_eq!(values.len(), arr.len(), "{name}");
|
||||
for (i, (v, w)) in values.iter().zip(&arr).enumerate() {
|
||||
assert!((v - w).abs() < 1e-12, "{name}[{i}]: {v} vs {w}");
|
||||
}
|
||||
}
|
||||
let grid: Vec<i32> = (0..37 * 21).collect();
|
||||
assert_eq!(
|
||||
file.dataset("grid_fl_shuf_gzip")
|
||||
.unwrap()
|
||||
.read_i32()
|
||||
.unwrap(),
|
||||
grid
|
||||
);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user