Huge chunks: selection tests with fill values, LZ4 over 256 MiB, wasm32 check
A selection of a chunked dataset with a fill value is compared with the full read in every index, through a map and through positioned reads. An LZ4 chunk larger than 256 MiB is bounded by the chunk size, not refused. The wasm package test reads the 4 GiB-chunk fixture and gets a clean error in every index. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
@@ -1491,7 +1491,7 @@ pub fn list_chunks_for_read_in<S: Storage + ?Sized>(
|
||||
let chunk_bytes = checked_chunk_byte_len(&chunk_dims, elem_size)?;
|
||||
if let Some(c) = chunks
|
||||
.iter()
|
||||
.find(|c| c.address != u64::MAX && c.chunk_size as usize != chunk_bytes)
|
||||
.find(|c| c.address != u64::MAX && c.chunk_size != chunk_bytes as u64)
|
||||
{
|
||||
return Err(FormatError::ChunkedReadError(format!(
|
||||
"incorrect chunk size returned from index for unfiltered chunk at {:?}: \
|
||||
@@ -2470,7 +2470,7 @@ mod tests {
|
||||
// Entries: key[i], child[i] pairs, then final key
|
||||
for chunk in chunks {
|
||||
// Key: chunk_size(4) + filter_mask(4) + ndims offsets
|
||||
buf.extend_from_slice(&chunk.chunk_size.to_le_bytes());
|
||||
buf.extend_from_slice(&(chunk.chunk_size as u32).to_le_bytes());
|
||||
buf.extend_from_slice(&chunk.filter_mask.to_le_bytes());
|
||||
for d in 0..ndims {
|
||||
let off = if d < chunk.offsets.len() {
|
||||
|
||||
Reference in New Issue
Block a user