Read chunks of 4 GiB or more in every chunk index

ChunkInfo::chunk_size (and ChunkMapping::file_size) are u64: sizes past
u32 were truncated for Single Chunk, Implicit, Fixed and Extensible Array
indexes, and a v2 B-tree index refused them. A selection of a chunked
dataset with a non-default fill value is read over a box of fill values
instead of a full read, an unfiltered chunk of a file that is not in memory
is read row by row, and an intermediate deflate stage no longer reserves the
chunk's whole bound.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-28 23:44:48 -05:00
co-authored by Claude Opus 5.5
parent 01d2a5dc5d
commit ac7871fff5
16 changed files with 604 additions and 43 deletions
+4 -4
View File
@@ -383,7 +383,7 @@ fn parse_fa_element(
offset_size: u8,
element_size: u8,
chunk_byte_size: u64,
) -> Result<Option<(u64, u32, u32)>, FormatError> {
) -> Result<Option<(u64, u64, u32)>, FormatError> {
let os = offset_size as usize;
if client_id == 0 {
// Non-filtered: element is just the chunk address.
@@ -393,7 +393,7 @@ fn parse_fa_element(
return Ok(None);
}
let address = read_offset(file_data, abs, offset_size)?;
Ok(Some((address, chunk_byte_size as u32, 0)))
Ok(Some((address, chunk_byte_size, 0)))
} else {
// Filtered: address(offset_size) + chunk_size(variable) + filter_mask(4)
let es = element_size as usize;
@@ -418,7 +418,7 @@ fn parse_fa_element(
file_data[fm_off + 2],
file_data[fm_off + 3],
]);
Ok(Some((address, chunk_size as u32, filter_mask)))
Ok(Some((address, chunk_size, filter_mask)))
}
}
@@ -688,7 +688,7 @@ mod tests {
assert_eq!(c.address, base_addr + i as u64 * chunk_byte_size as u64);
assert_eq!(c.offsets, vec![i as u64 * 20]);
assert_eq!(c.filter_mask, 0);
assert_eq!(c.chunk_size, chunk_byte_size as u32);
assert_eq!(c.chunk_size, chunk_byte_size as u64);
}
}