fix(format): read partial edge chunks stored unfiltered
Layout message v4 flag bit 0 (H5D_CHUNK_DONT_FILTER_PARTIAL_CHUNKS, set with H5Pset_chunk_opts) makes libhdf5 store every chunk that extends past the dataset's extent without the filter pipeline, while its filter mask still reads 0. The parser ignored the flag, so readers tried to inflate raw bytes: libhdf5's own h5fc_edge_v3.h5 failed with "deflate: ... unknown compression method". DataLayout::Chunked gains dont_filter_partial_edge_chunks (always false for v3), and list_chunks — the one place every read path gets its chunk list from — marks such partial chunks as having skipped every filter, so the full, cached, indexed, parallel and selection readers all copy them as-is. chunked_write.rs gets `..` in one exhaustive test pattern for the new field. Regression: libhdf5_edge_chunk_fixture_reads (h5fc_edge_v3.h5 from the HDF5 tools test files, committed as a 2.5 KB fixture), and h5py_unfiltered_partial_edge_chunks_read (the flag set through h5py's bundled libhdf5 via ctypes, as h5py has no binding for it: fixed array, extensible array and B-tree v2 indexes, 1-D and 2-D, plus a hyperslab of the last chunk), and v4_chunked_dont_filter_partial_edge_chunks_flag. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
@@ -519,6 +519,7 @@ pub fn list_chunks(
|
||||
addr_opt,
|
||||
single_filtered_size,
|
||||
single_filter_mask,
|
||||
unfiltered_edges,
|
||||
) = match layout {
|
||||
DataLayout::Chunked {
|
||||
chunk_dimensions,
|
||||
@@ -527,6 +528,7 @@ pub fn list_chunks(
|
||||
chunk_index_type,
|
||||
single_chunk_filtered_size,
|
||||
single_chunk_filter_mask,
|
||||
dont_filter_partial_edge_chunks,
|
||||
} => (
|
||||
chunk_dimensions,
|
||||
*version,
|
||||
@@ -534,6 +536,7 @@ pub fn list_chunks(
|
||||
*btree_address,
|
||||
*single_chunk_filtered_size,
|
||||
*single_chunk_filter_mask,
|
||||
*dont_filter_partial_edge_chunks,
|
||||
),
|
||||
_ => {
|
||||
return Err(FormatError::ChunkedReadError(
|
||||
@@ -566,7 +569,7 @@ pub fn list_chunks(
|
||||
}
|
||||
|
||||
// Collect chunks based on version and index type
|
||||
let chunks = match (version, chunk_index_type) {
|
||||
let mut chunks = match (version, chunk_index_type) {
|
||||
(3, _) => {
|
||||
let ndims = chunk_dimensions.len(); // rank+1
|
||||
collect_chunk_info(file_data, addr, ndims, offset_size, length_size)?
|
||||
@@ -645,6 +648,23 @@ pub fn list_chunks(
|
||||
}
|
||||
};
|
||||
|
||||
// With "don't filter partial edge chunks", a chunk that extends past the
|
||||
// dataset's extent is stored raw while its filter mask still reads 0.
|
||||
// Mark every filter skipped so all read paths copy it as-is.
|
||||
if unfiltered_edges {
|
||||
for chunk in &mut chunks {
|
||||
let partial = chunk
|
||||
.offsets
|
||||
.iter()
|
||||
.zip(&chunk_dims)
|
||||
.zip(&ds_dims)
|
||||
.any(|((&off, &cd), &dd)| off.saturating_add(cd as u64) > dd as u64);
|
||||
if partial {
|
||||
chunk.filter_mask = u32::MAX;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok((chunks, chunk_dims))
|
||||
}
|
||||
|
||||
@@ -1829,6 +1849,7 @@ mod tests {
|
||||
chunk_index_type: None,
|
||||
single_chunk_filtered_size: None,
|
||||
single_chunk_filter_mask: None,
|
||||
dont_filter_partial_edge_chunks: false,
|
||||
};
|
||||
|
||||
let dataspace = Dataspace {
|
||||
@@ -1852,6 +1873,7 @@ mod tests {
|
||||
chunk_index_type: None,
|
||||
single_chunk_filtered_size: None,
|
||||
single_chunk_filter_mask: None,
|
||||
dont_filter_partial_edge_chunks: false,
|
||||
};
|
||||
let dataspace = Dataspace {
|
||||
space_type: DataspaceType::Simple,
|
||||
@@ -2009,6 +2031,7 @@ mod tests {
|
||||
chunk_index_type: None,
|
||||
single_chunk_filtered_size: None,
|
||||
single_chunk_filter_mask: None,
|
||||
dont_filter_partial_edge_chunks: false,
|
||||
};
|
||||
let dataspace = Dataspace {
|
||||
space_type: DataspaceType::Simple,
|
||||
@@ -2091,6 +2114,7 @@ mod tests {
|
||||
chunk_index_type: None,
|
||||
single_chunk_filtered_size: None,
|
||||
single_chunk_filter_mask: None,
|
||||
dont_filter_partial_edge_chunks: false,
|
||||
};
|
||||
let dataspace = Dataspace {
|
||||
space_type: DataspaceType::Simple,
|
||||
@@ -2253,6 +2277,7 @@ mod tests {
|
||||
chunk_index_type: Some(1),
|
||||
single_chunk_filtered_size: None,
|
||||
single_chunk_filter_mask: None,
|
||||
dont_filter_partial_edge_chunks: false,
|
||||
};
|
||||
let dataspace = Dataspace {
|
||||
space_type: DataspaceType::Simple,
|
||||
|
||||
@@ -1314,6 +1314,7 @@ mod tests {
|
||||
chunk_index_type,
|
||||
single_chunk_filtered_size,
|
||||
single_chunk_filter_mask,
|
||||
..
|
||||
} => {
|
||||
assert_eq!(version, 4);
|
||||
assert_eq!(chunk_index_type, Some(1));
|
||||
|
||||
@@ -53,6 +53,11 @@ pub enum DataLayout {
|
||||
single_chunk_filtered_size: Option<u64>,
|
||||
/// Filter mask for v4 single chunk with filters.
|
||||
single_chunk_filter_mask: Option<u32>,
|
||||
/// Layout v4 flag bit 0 (`H5D_CHUNK_DONT_FILTER_PARTIAL_CHUNKS`):
|
||||
/// partial edge chunks — those extending past the dataset's current
|
||||
/// extent in some dimension — are stored without the filter pipeline,
|
||||
/// even though their filter mask is 0. Always `false` for v3.
|
||||
dont_filter_partial_edge_chunks: bool,
|
||||
},
|
||||
/// Virtual dataset layout (v4 only).
|
||||
Virtual {
|
||||
@@ -322,6 +327,7 @@ impl DataLayout {
|
||||
chunk_index_type: None,
|
||||
single_chunk_filtered_size: None,
|
||||
single_chunk_filter_mask: None,
|
||||
dont_filter_partial_edge_chunks: false,
|
||||
})
|
||||
}
|
||||
_ => Err(FormatError::InvalidLayoutClass(layout_class)),
|
||||
@@ -505,6 +511,7 @@ impl DataLayout {
|
||||
chunk_index_type: Some(chunk_index_type),
|
||||
single_chunk_filtered_size,
|
||||
single_chunk_filter_mask,
|
||||
dont_filter_partial_edge_chunks: flags & 0x01 != 0,
|
||||
})
|
||||
}
|
||||
3 => {
|
||||
@@ -602,6 +609,7 @@ mod tests {
|
||||
chunk_index_type: None,
|
||||
single_chunk_filtered_size: None,
|
||||
single_chunk_filter_mask: None,
|
||||
dont_filter_partial_edge_chunks: false,
|
||||
}
|
||||
);
|
||||
}
|
||||
@@ -679,10 +687,35 @@ mod tests {
|
||||
chunk_index_type: Some(1),
|
||||
single_chunk_filtered_size: None,
|
||||
single_chunk_filter_mask: None,
|
||||
dont_filter_partial_edge_chunks: false,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn v4_chunked_dont_filter_partial_edge_chunks_flag() {
|
||||
let mut buf = vec![4u8, 2]; // version=4, class=2
|
||||
buf.push(0x01); // flags bit 0 = don't filter partial edge chunks
|
||||
buf.push(2); // dimensionality=2
|
||||
buf.push(4); // dim_size_encoded_length=4
|
||||
buf.extend_from_slice(&5u32.to_le_bytes());
|
||||
buf.extend_from_slice(&4u32.to_le_bytes());
|
||||
buf.push(3); // Fixed Array
|
||||
buf.push(10); // max_dblk_page_nelmts_bits
|
||||
buf.extend_from_slice(&0x3000u64.to_le_bytes());
|
||||
match DataLayout::parse(&buf, 8, 8).unwrap() {
|
||||
DataLayout::Chunked {
|
||||
dont_filter_partial_edge_chunks,
|
||||
btree_address,
|
||||
..
|
||||
} => {
|
||||
assert!(dont_filter_partial_edge_chunks);
|
||||
assert_eq!(btree_address, Some(0x3000));
|
||||
}
|
||||
other => panic!("expected Chunked, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn v4_chunked_single_chunk_with_filters() {
|
||||
let mut buf = vec![4u8, 2]; // version=4, class=2
|
||||
@@ -705,6 +738,7 @@ mod tests {
|
||||
chunk_index_type: Some(1),
|
||||
single_chunk_filtered_size: Some(1024),
|
||||
single_chunk_filter_mask: Some(0),
|
||||
dont_filter_partial_edge_chunks: false,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user