diff --git a/crates/clawhdf5-format/src/filters.rs b/crates/clawhdf5-format/src/filters.rs index 94fa15f..5d9fded 100644 --- a/crates/clawhdf5-format/src/filters.rs +++ b/crates/clawhdf5-format/src/filters.rs @@ -28,7 +28,20 @@ pub fn decompress_chunk( ) -> Result, FormatError> { let mut data = compressed.to_vec(); - for filter in pipeline.filters.iter().rev() { + for (i, filter) in pipeline.filters.iter().enumerate().rev() { + // Fletcher32 appends a 4-byte checksum on write, so a filter that + // follows it in the (forward) pipeline decodes to the chunk plus one + // checksum per earlier Fletcher32 — e.g. libhdf5's + // shuffle + fletcher32 + deflate (h5repack_filters.h5 `/dset_all`). + let chunk_size = if chunk_size == 0 { + 0 + } else { + let checksums = pipeline.filters[..i] + .iter() + .filter(|f| f.filter_id == FILTER_FLETCHER32) + .count(); + chunk_size.saturating_add(4 * checksums) + }; data = match filter.filter_id { FILTER_SHUFFLE => shuffle_decompress(&data, element_size as usize)?, // `chunk_size` is the expected decompressed size (shuffle/fletcher32 @@ -1642,6 +1655,40 @@ mod tests { } } + /// Fletcher32 before a compressor (libhdf5 applies filters in pipeline + /// order, so the compressor sees chunk + checksum): deflate's output is 4 + /// bytes over the chunk size, which we rejected as "deflate: output + /// exceeds size limit". Chunks from h5py/libhdf5, values from h5py. + #[test] + #[cfg(feature = "deflate")] + fn fletcher32_before_deflate_decodes() { + // h5py: set_fletcher32(); set_deflate(4); i32 0..100, chunks of 10. + let pipeline = FilterPipeline { + version: 2, + filters: vec![ + one_filter(FILTER_FLETCHER32, vec![]), + one_filter(FILTER_DEFLATE, vec![4]), + ], + }; + let raw = + unhex("785e936360609007620520560462252056066215205605623520560762c6483e3d00234501f0"); + let want: Vec = (30..40i32).flat_map(i32::to_le_bytes).collect(); + assert_eq!(decompress_chunk(&raw, &pipeline, 40, 4).unwrap(), want); + + // And our own writer's round trip through the same pipeline order. + let data: Vec = (0..400u32).map(|i| (i % 13) as u8).collect(); + let pipeline = FilterPipeline { + version: 2, + filters: vec![ + one_filter(FILTER_SHUFFLE, vec![4]), + one_filter(FILTER_FLETCHER32, vec![]), + one_filter(FILTER_DEFLATE, vec![9]), + ], + }; + let c = compress_chunk(&data, &pipeline, 4).unwrap(); + assert_eq!(decompress_chunk(&c, &pipeline, 400, 4).unwrap(), data); + } + /// `le_data.h5` scale-offset (D-scale, D = 3, fill -2.2) chunks, decoded /// bit for bit as libhdf5 does: single-precision arithmetic for `float` /// (we computed in f64 and rounded once, which was 1 ULP off for e.g.