Read chunks of 4 GiB or more in every chunk index

ChunkInfo::chunk_size (and ChunkMapping::file_size) are u64: sizes past
u32 were truncated for Single Chunk, Implicit, Fixed and Extensible Array
indexes, and a v2 B-tree index refused them. A selection of a chunked
dataset with a non-default fill value is read over a box of fill values
instead of a full read, an unfiltered chunk of a file that is not in memory
is read row by row, and an intermediate deflate stage no longer reserves the
chunk's whole bound.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-28 23:44:48 -05:00
co-authored by Claude Opus 5.5
parent 01d2a5dc5d
commit ac7871fff5
16 changed files with 604 additions and 43 deletions
@@ -173,7 +173,7 @@ fn crafted() -> (Vec<u8>, Chunked, Vec<ChunkInfo>) {
assert!(
chunks
.iter()
.all(|c| c.chunk_size == HUGE && c.address == blob)
.all(|c| c.chunk_size == u64::from(HUGE) && c.address == blob)
);
(bytes, ds, chunks)
}
@@ -313,7 +313,7 @@ fn large_reads_are_fetched_in_batches() {
let data: Vec<u8> = (0..2 * CHUNK).map(|i| (i % 251) as u8).collect();
let chunks: Vec<ChunkInfo> = (0..40u64)
.map(|i| ChunkInfo {
chunk_size: CHUNK as u32,
chunk_size: CHUNK as u64,
filter_mask: 0,
offsets: vec![i * CHUNK as u64],
address: (i % 2) * CHUNK as u64,