Read chunks of 4 GiB or more in every chunk index
ChunkInfo::chunk_size (and ChunkMapping::file_size) are u64: sizes past u32 were truncated for Single Chunk, Implicit, Fixed and Extensible Array indexes, and a v2 B-tree index refused them. A selection of a chunked dataset with a non-default fill value is read over a box of fill values instead of a full read, an unfiltered chunk of a file that is not in memory is read row by row, and an intermediate deflate stage no longer reserves the chunk's whole bound. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
@@ -383,7 +383,7 @@ fn parse_fa_element(
|
||||
offset_size: u8,
|
||||
element_size: u8,
|
||||
chunk_byte_size: u64,
|
||||
) -> Result<Option<(u64, u32, u32)>, FormatError> {
|
||||
) -> Result<Option<(u64, u64, u32)>, FormatError> {
|
||||
let os = offset_size as usize;
|
||||
if client_id == 0 {
|
||||
// Non-filtered: element is just the chunk address.
|
||||
@@ -393,7 +393,7 @@ fn parse_fa_element(
|
||||
return Ok(None);
|
||||
}
|
||||
let address = read_offset(file_data, abs, offset_size)?;
|
||||
Ok(Some((address, chunk_byte_size as u32, 0)))
|
||||
Ok(Some((address, chunk_byte_size, 0)))
|
||||
} else {
|
||||
// Filtered: address(offset_size) + chunk_size(variable) + filter_mask(4)
|
||||
let es = element_size as usize;
|
||||
@@ -418,7 +418,7 @@ fn parse_fa_element(
|
||||
file_data[fm_off + 2],
|
||||
file_data[fm_off + 3],
|
||||
]);
|
||||
Ok(Some((address, chunk_size as u32, filter_mask)))
|
||||
Ok(Some((address, chunk_size, filter_mask)))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -688,7 +688,7 @@ mod tests {
|
||||
assert_eq!(c.address, base_addr + i as u64 * chunk_byte_size as u64);
|
||||
assert_eq!(c.offsets, vec![i as u64 * 20]);
|
||||
assert_eq!(c.filter_mask, 0);
|
||||
assert_eq!(c.chunk_size, chunk_byte_size as u32);
|
||||
assert_eq!(c.chunk_size, chunk_byte_size as u64);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user