fix(format): read v1 chunk B-tree key offsets as 8 bytes

A type-1 (raw data chunk) B-tree key holds the chunk size, the filter
mask and one offset per dimension, and those offsets are always 8 bytes:
they are dataset coordinates, not file addresses. The reader used the
superblock's size-of-offsets for them, so in a file with 4-byte offsets
every key was misparsed. Unfiltered chunked datasets read as zeros (with
stray bytes where a misread address landed on data) and filtered ones
failed with "deflate: truncated stream".

Only the sibling and child addresses follow size-of-offsets now. The
unit-test B-tree builder wrote keys the same wrong way, which is why its
tests passed; it now matches the format.

Regression: h5py_four_byte_offsets_chunked_reads (h5py, set_sizes(4, 4)
and (4, 8); 1-D and 2-D, unfiltered and gzip) and the unit test
collect_chunks_with_four_byte_addresses.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-25 21:01:57 -05:00
co-authored by Claude Opus 5.5
parent 46203ea761
commit 9ea44d473d
2 changed files with 153 additions and 6 deletions
+48 -6
View File
@@ -223,6 +223,10 @@ pub fn collect_chunk_info(
collect_chunk_info_inner(file_data, btree_address, ndims, offset_size, length_size, 0)
}
/// Width of each chunk offset in a v1 chunk B-tree key, independent of the
/// file's size-of-offsets.
const CHUNK_KEY_OFFSET_SIZE: u8 = 8;
/// Maximum recursion depth for chunk B-tree traversal (malformed/cyclic data
/// protection), matching `btree_v1.rs`'s `MAX_BTREE_DEPTH`.
const MAX_CHUNK_BTREE_DEPTH: usize = 64;
@@ -260,8 +264,14 @@ fn collect_chunk_info_inner(
let mut pos = offset + 8 + os * 2; // skip left/right sibling
// Key size: chunk_size(4) + filter_mask(4) + ndims * offset_size
let key_size = 4 + 4 + ndims * os;
// Key: chunk_size(4) + filter_mask(4) + one offset per dimension. The
// offsets are always 8 bytes each — they are dataset coordinates, not file
// addresses, so they do not follow the superblock's size-of-offsets (only
// the sibling and child addresses do).
let key_size = ndims
.checked_mul(CHUNK_KEY_OFFSET_SIZE as usize)
.and_then(|n| n.checked_add(8))
.ok_or_else(|| FormatError::ChunkedReadError("chunk key too large".into()))?;
if node_level == 0 {
// Leaf node: keys and children interleaved
@@ -287,8 +297,8 @@ fn collect_chunk_info_inner(
let mut offsets = Vec::with_capacity(ndims);
let mut kp = pos + 8;
for _ in 0..ndims {
offsets.push(read_offset(file_data, kp, offset_size)?);
kp += os;
offsets.push(read_offset(file_data, kp, CHUNK_KEY_OFFSET_SIZE)?);
kp += CHUNK_KEY_OFFSET_SIZE as usize;
}
pos += key_size;
@@ -1592,7 +1602,8 @@ mod tests {
} else {
0
};
write_offset(&mut buf, off, offset_size);
// Key offsets are always 8 bytes (they are coordinates).
write_offset(&mut buf, off, 8);
}
// Child: address
write_offset(&mut buf, chunk.address, offset_size);
@@ -1602,7 +1613,7 @@ mod tests {
buf.extend_from_slice(&0u32.to_le_bytes()); // chunk_size
buf.extend_from_slice(&0u32.to_le_bytes()); // filter_mask
for _ in 0..ndims {
write_offset(&mut buf, u64::MAX, offset_size);
write_offset(&mut buf, u64::MAX, 8);
}
buf
@@ -1680,6 +1691,37 @@ mod tests {
assert_eq!(result[2].address, 0x300);
}
#[test]
fn collect_chunks_with_four_byte_addresses() {
// Sibling and child addresses are 4 bytes; the key offsets stay 8.
let ndims = 3;
let os: u8 = 4;
let chunks = vec![
ChunkInfo {
chunk_size: 80,
filter_mask: 2,
offsets: vec![0, 5, 0],
address: 0x1000,
},
ChunkInfo {
chunk_size: 96,
filter_mask: 0,
offsets: vec![8, 10, 0],
address: 0x2000,
},
];
let btree = build_chunk_btree_leaf(&chunks, ndims, os);
assert_eq!(btree.len(), 8 + 2 * 4 + 2 * (8 + 3 * 8 + 4) + (8 + 3 * 8));
let result = collect_chunk_info(&btree, 0, ndims, os, os).unwrap();
assert_eq!(result.len(), 2);
for (got, want) in result.iter().zip(&chunks) {
assert_eq!(got.offsets, want.offsets);
assert_eq!(got.address, want.address);
assert_eq!(got.chunk_size, want.chunk_size);
assert_eq!(got.filter_mask, want.filter_mask);
}
}
#[test]
fn collect_empty_btree() {
let ndims = 2;