quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix(format): honour each bit of a chunk's filter mask

Browse Source 
A chunk's filter mask has one bit per pipeline filter; bit i set means
filter i was not applied to that chunk (an optional filter that
declined, or a direct chunk write). Every read path treated any nonzero
mask as "no filters applied" and returned the stored bytes, so a chunk
that skipped only gzip in a shuffle+gzip pipeline came back still
shuffled (h5py write_direct_chunk with filter_mask=0b10: 8 of 32 values
wrong).

decompress_chunk_masked undoes the filters the mask leaves set and skips
the rest; an unsupported filter is no longer an error when the chunk
skipped it. The full, cached, sweep, indexed, parallel and selection
(partial_read) paths all use it, and a chunk is copied straight from the
file only when every filter was skipped. decompress_chunk is the mask-0
case.

Regression: h5py_partial_filter_mask_skips_only_masked_filters (1-D
shuffle+gzip with masks 0, 0b10 and 0b11; 2-D with 0b01; full and
hyperslab reads) and filter_mask_skips_only_the_masked_filters.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-25 21:05:07 -05:00
co-authored by Claude Opus 5.5
parent 9ea44d473d
commit 585e14d5e2
5 changed files with 244 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/parallel_read.rs
+22 -16
View File
@@ -10,7 +10,7 @@
use crate::chunked_read::ChunkInfo;
use crate::error::FormatError;
use crate::filter_pipeline::FilterPipeline;
use crate::filters::decompress_chunk;
use crate::filters::decompress_chunk_masked;
use crate::lane_partition::{self, LaneStats, PartitionStats};
/// Threshold: only use parallel decompression when chunk count exceeds this.
@@ -84,11 +84,13 @@ pub fn decompress_chunks_lane_partitioned(
}
let raw_chunk = &file_data[c_addr..c_addr + size];
let decompressed = if chunk_info.filter_mask == 0 {
decompress_chunk(raw_chunk, pipeline, chunk_total_bytes, element_size)?
} else {
raw_chunk.to_vec()
};
let decompressed = decompress_chunk_masked(
raw_chunk,
pipeline,
chunk_total_bytes,
element_size,
chunk_info.filter_mask,
)?;
stats.chunks_processed += 1;
stats.compressed_bytes += size as u64;
@@ -158,11 +160,13 @@ pub fn decompress_chunks_parallel(
}
let raw_chunk = &file_data[c_addr..c_addr + size];
let decompressed = if chunk_info.filter_mask == 0 {
decompress_chunk(raw_chunk, pipeline, chunk_total_bytes, element_size)?
} else {
raw_chunk.to_vec()
};
let decompressed = decompress_chunk_masked(
raw_chunk,
pipeline,
chunk_total_bytes,
element_size,
chunk_info.filter_mask,
)?;
Ok(DecompressedChunk {
index,
@@ -200,11 +204,13 @@ pub fn decompress_chunks_sequential(
let raw_chunk = &file_data[c_addr..c_addr + size];
let decompressed = if let Some(pl) = pipeline {
if chunk_info.filter_mask == 0 {
decompress_chunk(raw_chunk, pl, chunk_total_bytes, element_size)?
} else {
raw_chunk.to_vec()
}
decompress_chunk_masked(
raw_chunk,
pl,
chunk_total_bytes,
element_size,
chunk_info.filter_mask,
)?
} else {
raw_chunk.to_vec()
};