fix(format): read partial edge chunks stored unfiltered

Layout message v4 flag bit 0 (H5D_CHUNK_DONT_FILTER_PARTIAL_CHUNKS, set
with H5Pset_chunk_opts) makes libhdf5 store every chunk that extends past
the dataset's extent without the filter pipeline, while its filter mask
still reads 0. The parser ignored the flag, so readers tried to inflate
raw bytes: libhdf5's own h5fc_edge_v3.h5 failed with "deflate: ...
unknown compression method".

DataLayout::Chunked gains dont_filter_partial_edge_chunks (always false
for v3), and list_chunks — the one place every read path gets its chunk
list from — marks such partial chunks as having skipped every filter, so
the full, cached, indexed, parallel and selection readers all copy them
as-is. chunked_write.rs gets `..` in one exhaustive test pattern for the
new field.

Regression: libhdf5_edge_chunk_fixture_reads (h5fc_edge_v3.h5 from the
HDF5 tools test files, committed as a 2.5 KB fixture), and
h5py_unfiltered_partial_edge_chunks_read (the flag set through h5py's
bundled libhdf5 via ctypes, as h5py has no binding for it: fixed array,
extensible array and B-tree v2 indexes, 1-D and 2-D, plus a hyperslab
of the last chunk), and v4_chunked_dont_filter_partial_edge_chunks_flag.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-25 21:08:36 -05:00
co-authored by Claude Opus 5.5
parent e162c013fd
commit d074385944
5 changed files with 167 additions and 1 deletions
+34
View File
@@ -53,6 +53,11 @@ pub enum DataLayout {
single_chunk_filtered_size: Option<u64>,
/// Filter mask for v4 single chunk with filters.
single_chunk_filter_mask: Option<u32>,
/// Layout v4 flag bit 0 (`H5D_CHUNK_DONT_FILTER_PARTIAL_CHUNKS`):
/// partial edge chunks — those extending past the dataset's current
/// extent in some dimension — are stored without the filter pipeline,
/// even though their filter mask is 0. Always `false` for v3.
dont_filter_partial_edge_chunks: bool,
},
/// Virtual dataset layout (v4 only).
Virtual {
@@ -322,6 +327,7 @@ impl DataLayout {
chunk_index_type: None,
single_chunk_filtered_size: None,
single_chunk_filter_mask: None,
dont_filter_partial_edge_chunks: false,
})
}
_ => Err(FormatError::InvalidLayoutClass(layout_class)),
@@ -505,6 +511,7 @@ impl DataLayout {
chunk_index_type: Some(chunk_index_type),
single_chunk_filtered_size,
single_chunk_filter_mask,
dont_filter_partial_edge_chunks: flags & 0x01 != 0,
})
}
3 => {
@@ -602,6 +609,7 @@ mod tests {
chunk_index_type: None,
single_chunk_filtered_size: None,
single_chunk_filter_mask: None,
dont_filter_partial_edge_chunks: false,
}
);
}
@@ -679,10 +687,35 @@ mod tests {
chunk_index_type: Some(1),
single_chunk_filtered_size: None,
single_chunk_filter_mask: None,
dont_filter_partial_edge_chunks: false,
}
);
}
#[test]
fn v4_chunked_dont_filter_partial_edge_chunks_flag() {
let mut buf = vec![4u8, 2]; // version=4, class=2
buf.push(0x01); // flags bit 0 = don't filter partial edge chunks
buf.push(2); // dimensionality=2
buf.push(4); // dim_size_encoded_length=4
buf.extend_from_slice(&5u32.to_le_bytes());
buf.extend_from_slice(&4u32.to_le_bytes());
buf.push(3); // Fixed Array
buf.push(10); // max_dblk_page_nelmts_bits
buf.extend_from_slice(&0x3000u64.to_le_bytes());
match DataLayout::parse(&buf, 8, 8).unwrap() {
DataLayout::Chunked {
dont_filter_partial_edge_chunks,
btree_address,
..
} => {
assert!(dont_filter_partial_edge_chunks);
assert_eq!(btree_address, Some(0x3000));
}
other => panic!("expected Chunked, got {other:?}"),
}
}
#[test]
fn v4_chunked_single_chunk_with_filters() {
let mut buf = vec![4u8, 2]; // version=4, class=2
@@ -705,6 +738,7 @@ mod tests {
chunk_index_type: Some(1),
single_chunk_filtered_size: Some(1024),
single_chunk_filter_mask: Some(0),
dont_filter_partial_edge_chunks: false,
}
);
}