feat(format): read chunked datasets indexed by a version-2 B-tree
With libver='latest', a chunked dataset with two or more unlimited dimensions indexes its chunks with a v2 B-tree (layout v4, index type 5). Reading one failed with "unsupported chunked layout version=4, index_type=Some(5)". read_btree_v2_chunks decodes record types 10 (address + scaled offsets) and 11 (address, stored size, filter mask, scaled offsets). The width of the stored-size field is taken from the record size the tree header declares rather than re-deriving the library's formula. Scaled offsets are multiplied back by the chunk dimensions with overflow checks. The chunk-index dispatch existed four times (uncached, cached, sweep and indexed readers). The three copies outside list_chunks now call it, so every read path — and fill-value handling and partial reads — supports every index type from one place. h5py interop test: plain, gzip+shuffle, a 2500-chunk tree with internal nodes, a sparse dataset with a fill value, and a strided hyperslab. Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
osobh
co-authored by
Claude Fable 5.1
Claude Fable 5.1
parent
c6a7bbfc67
commit
d668e45ab5
@@ -362,6 +362,112 @@ pub fn generate_implicit_chunks(
|
||||
}
|
||||
|
||||
/// Read a chunked dataset, decompressing chunks as needed.
|
||||
/// B-tree v2 record types used for chunk indexing.
|
||||
const BT2_CHUNK_UNFILTERED: u8 = 10;
|
||||
const BT2_CHUNK_FILTERED: u8 = 11;
|
||||
|
||||
/// Chunks indexed by a version-2 B-tree (layout v4, index type 5).
|
||||
///
|
||||
/// Record layouts (all little endian):
|
||||
/// * type 10, unfiltered: address, then one 8-byte *scaled* offset per
|
||||
/// dimension (offset / chunk dimension);
|
||||
/// * type 11, filtered: address, stored chunk size (a variable number of
|
||||
/// bytes), 4-byte filter mask, then the scaled offsets.
|
||||
///
|
||||
/// The width of the stored-size field depends on the largest possible chunk;
|
||||
/// rather than re-derive the library's formula it is taken from the record
|
||||
/// size the tree header declares, which is what actually governs the bytes.
|
||||
fn read_btree_v2_chunks(
|
||||
file_data: &[u8],
|
||||
addr: u64,
|
||||
chunk_dims: &[usize],
|
||||
elem_size: usize,
|
||||
offset_size: u8,
|
||||
length_size: u8,
|
||||
) -> Result<Vec<ChunkInfo>, FormatError> {
|
||||
use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records};
|
||||
|
||||
let bad = |what: &str| FormatError::ChunkedReadError(format!("B-tree v2 chunk index: {what}"));
|
||||
let header = BTreeV2Header::parse(file_data, addr as usize, offset_size, length_size)?;
|
||||
let rank = chunk_dims.len();
|
||||
let os = offset_size as usize;
|
||||
let record_size = header.record_size as usize;
|
||||
let size_len = match header.tree_type {
|
||||
BT2_CHUNK_UNFILTERED => {
|
||||
if record_size != os + 8 * rank {
|
||||
return Err(bad("unexpected record size for unfiltered chunks"));
|
||||
}
|
||||
0
|
||||
}
|
||||
BT2_CHUNK_FILTERED => {
|
||||
let fixed = os + 4 + 8 * rank;
|
||||
let size_len = record_size
|
||||
.checked_sub(fixed)
|
||||
.ok_or_else(|| bad("record too small"))?;
|
||||
if !(1..=8).contains(&size_len) {
|
||||
return Err(bad("implausible chunk-size field width"));
|
||||
}
|
||||
size_len
|
||||
}
|
||||
_ => return Err(bad("tree is not a chunk index")),
|
||||
};
|
||||
let unfiltered_bytes = checked_chunk_byte_len(chunk_dims, elem_size)?;
|
||||
let unfiltered_bytes =
|
||||
u32::try_from(unfiltered_bytes).map_err(|_| bad("chunk larger than 4 GiB"))?;
|
||||
|
||||
let records = collect_btree_v2_records(file_data, &header, offset_size, length_size)?;
|
||||
let mut chunks = Vec::with_capacity(records.len());
|
||||
for record in &records {
|
||||
let data = record.data.as_slice();
|
||||
if data.len() < record_size {
|
||||
return Err(bad("truncated record"));
|
||||
}
|
||||
let address = read_offset(data, 0, offset_size)?;
|
||||
let mut pos = os;
|
||||
let (chunk_size, filter_mask) = if size_len == 0 {
|
||||
(unfiltered_bytes, 0)
|
||||
} else {
|
||||
let mut size = 0u64;
|
||||
for (i, &b) in data[pos..pos + size_len].iter().enumerate() {
|
||||
size |= u64::from(b) << (8 * i);
|
||||
}
|
||||
pos += size_len;
|
||||
let mask = u32::from_le_bytes([data[pos], data[pos + 1], data[pos + 2], data[pos + 3]]);
|
||||
pos += 4;
|
||||
(
|
||||
u32::try_from(size).map_err(|_| bad("stored chunk larger than 4 GiB"))?,
|
||||
mask,
|
||||
)
|
||||
};
|
||||
let mut offsets = Vec::with_capacity(rank);
|
||||
for &dim in chunk_dims {
|
||||
let scaled = u64::from_le_bytes([
|
||||
data[pos],
|
||||
data[pos + 1],
|
||||
data[pos + 2],
|
||||
data[pos + 3],
|
||||
data[pos + 4],
|
||||
data[pos + 5],
|
||||
data[pos + 6],
|
||||
data[pos + 7],
|
||||
]);
|
||||
pos += 8;
|
||||
offsets.push(
|
||||
scaled
|
||||
.checked_mul(dim as u64)
|
||||
.ok_or_else(|| bad("chunk offset overflows"))?,
|
||||
);
|
||||
}
|
||||
chunks.push(ChunkInfo {
|
||||
chunk_size,
|
||||
filter_mask,
|
||||
offsets,
|
||||
address,
|
||||
});
|
||||
}
|
||||
Ok(chunks)
|
||||
}
|
||||
|
||||
/// Every allocated chunk of a chunked dataset, for any supported chunk index,
|
||||
/// plus the spatial chunk dimensions. Chunks the file never allocated (sparse
|
||||
/// datasets) are simply absent from the list.
|
||||
@@ -487,6 +593,18 @@ pub fn list_chunks(
|
||||
length_size,
|
||||
)?
|
||||
}
|
||||
(4, Some(5)) => {
|
||||
// Version-2 B-tree: what the library uses for a dataset with two
|
||||
// or more unlimited dimensions.
|
||||
read_btree_v2_chunks(
|
||||
file_data,
|
||||
addr,
|
||||
&chunk_dims,
|
||||
elem_size,
|
||||
offset_size,
|
||||
length_size,
|
||||
)?
|
||||
}
|
||||
(v, idx) => {
|
||||
return Err(FormatError::ChunkedReadError(format!(
|
||||
"unsupported chunked layout version={v}, index_type={idx:?}"
|
||||
@@ -629,29 +747,12 @@ pub fn read_chunked_data_cached(
|
||||
length_size: u8,
|
||||
cache: &ChunkCache,
|
||||
) -> Result<Vec<u8>, FormatError> {
|
||||
let (
|
||||
chunk_dimensions,
|
||||
version,
|
||||
chunk_index_type,
|
||||
addr_opt,
|
||||
single_filtered_size,
|
||||
single_filter_mask,
|
||||
) = match layout {
|
||||
let (chunk_dimensions, addr_opt) = match layout {
|
||||
DataLayout::Chunked {
|
||||
chunk_dimensions,
|
||||
btree_address,
|
||||
version,
|
||||
chunk_index_type,
|
||||
single_chunk_filtered_size,
|
||||
single_chunk_filter_mask,
|
||||
} => (
|
||||
chunk_dimensions,
|
||||
*version,
|
||||
*chunk_index_type,
|
||||
*btree_address,
|
||||
*single_chunk_filtered_size,
|
||||
*single_chunk_filter_mask,
|
||||
),
|
||||
..
|
||||
} => (chunk_dimensions, *btree_address),
|
||||
_ => {
|
||||
return Err(FormatError::ChunkedReadError(
|
||||
"expected chunked layout".into(),
|
||||
@@ -688,69 +789,14 @@ pub fn read_chunked_data_cached(
|
||||
|
||||
// Populate chunk index on first access
|
||||
if !cache.has_index() {
|
||||
let chunks = match (version, chunk_index_type) {
|
||||
(3, _) => collect_chunk_info(file_data, addr, ndims, offset_size, length_size)?,
|
||||
(4, Some(1)) => {
|
||||
let chunk_byte_size = checked_chunk_byte_len(&chunk_dims, elem_size)?;
|
||||
let (csize, fmask) = if let Some(fs) = single_filtered_size {
|
||||
(fs as u32, single_filter_mask.unwrap_or(0))
|
||||
} else {
|
||||
(chunk_byte_size as u32, 0)
|
||||
};
|
||||
vec![ChunkInfo {
|
||||
chunk_size: csize,
|
||||
filter_mask: fmask,
|
||||
offsets: vec![0u64; rank],
|
||||
address: addr,
|
||||
}]
|
||||
}
|
||||
(4, Some(2)) => {
|
||||
let spatial_chunk_dims: &[u32] = &chunk_dimensions[..rank];
|
||||
generate_implicit_chunks(
|
||||
addr,
|
||||
&dataspace.dimensions,
|
||||
spatial_chunk_dims,
|
||||
elem_size as u32,
|
||||
)
|
||||
}
|
||||
(4, Some(3)) => {
|
||||
let spatial_chunk_dims: &[u32] = &chunk_dimensions[..rank];
|
||||
let header =
|
||||
FixedArrayHeader::parse(file_data, addr as usize, offset_size, length_size)?;
|
||||
read_fixed_array_chunks(
|
||||
file_data,
|
||||
&header,
|
||||
&dataspace.dimensions,
|
||||
spatial_chunk_dims,
|
||||
elem_size as u32,
|
||||
offset_size,
|
||||
length_size,
|
||||
)?
|
||||
}
|
||||
(4, Some(4)) => {
|
||||
let spatial_chunk_dims: &[u32] = &chunk_dimensions[..rank];
|
||||
let header = ExtensibleArrayHeader::parse(
|
||||
file_data,
|
||||
addr as usize,
|
||||
offset_size,
|
||||
length_size,
|
||||
)?;
|
||||
read_extensible_array_chunks(
|
||||
file_data,
|
||||
&header,
|
||||
&dataspace.dimensions,
|
||||
spatial_chunk_dims,
|
||||
elem_size as u32,
|
||||
offset_size,
|
||||
length_size,
|
||||
)?
|
||||
}
|
||||
(v, idx) => {
|
||||
return Err(FormatError::ChunkedReadError(format!(
|
||||
"unsupported chunked layout version={v}, index_type={idx:?}"
|
||||
)));
|
||||
}
|
||||
};
|
||||
let (chunks, _) = list_chunks(
|
||||
file_data,
|
||||
layout,
|
||||
dataspace,
|
||||
elem_size,
|
||||
offset_size,
|
||||
length_size,
|
||||
)?;
|
||||
cache.populate_index(&chunks, rank);
|
||||
}
|
||||
|
||||
@@ -985,29 +1031,12 @@ pub fn read_chunked_data_sweep(
|
||||
cache: &ChunkCache,
|
||||
sweep: &mut SweepContext,
|
||||
) -> Result<Vec<u8>, FormatError> {
|
||||
let (
|
||||
chunk_dimensions,
|
||||
version,
|
||||
chunk_index_type,
|
||||
addr_opt,
|
||||
single_filtered_size,
|
||||
single_filter_mask,
|
||||
) = match layout {
|
||||
let (chunk_dimensions, addr_opt) = match layout {
|
||||
DataLayout::Chunked {
|
||||
chunk_dimensions,
|
||||
btree_address,
|
||||
version,
|
||||
chunk_index_type,
|
||||
single_chunk_filtered_size,
|
||||
single_chunk_filter_mask,
|
||||
} => (
|
||||
chunk_dimensions,
|
||||
*version,
|
||||
*chunk_index_type,
|
||||
*btree_address,
|
||||
*single_chunk_filtered_size,
|
||||
*single_chunk_filter_mask,
|
||||
),
|
||||
..
|
||||
} => (chunk_dimensions, *btree_address),
|
||||
_ => {
|
||||
return Err(FormatError::ChunkedReadError(
|
||||
"expected chunked layout".into(),
|
||||
@@ -1044,69 +1073,14 @@ pub fn read_chunked_data_sweep(
|
||||
|
||||
// Populate chunk index on first access
|
||||
if !cache.has_index() {
|
||||
let chunks = match (version, chunk_index_type) {
|
||||
(3, _) => collect_chunk_info(file_data, addr, ndims, offset_size, length_size)?,
|
||||
(4, Some(1)) => {
|
||||
let chunk_byte_size = checked_chunk_byte_len(&chunk_dims, elem_size)?;
|
||||
let (csize, fmask) = if let Some(fs) = single_filtered_size {
|
||||
(fs as u32, single_filter_mask.unwrap_or(0))
|
||||
} else {
|
||||
(chunk_byte_size as u32, 0)
|
||||
};
|
||||
vec![ChunkInfo {
|
||||
chunk_size: csize,
|
||||
filter_mask: fmask,
|
||||
offsets: vec![0u64; rank],
|
||||
address: addr,
|
||||
}]
|
||||
}
|
||||
(4, Some(2)) => {
|
||||
let spatial_chunk_dims: &[u32] = &chunk_dimensions[..rank];
|
||||
generate_implicit_chunks(
|
||||
addr,
|
||||
&dataspace.dimensions,
|
||||
spatial_chunk_dims,
|
||||
elem_size as u32,
|
||||
)
|
||||
}
|
||||
(4, Some(3)) => {
|
||||
let spatial_chunk_dims: &[u32] = &chunk_dimensions[..rank];
|
||||
let header =
|
||||
FixedArrayHeader::parse(file_data, addr as usize, offset_size, length_size)?;
|
||||
read_fixed_array_chunks(
|
||||
file_data,
|
||||
&header,
|
||||
&dataspace.dimensions,
|
||||
spatial_chunk_dims,
|
||||
elem_size as u32,
|
||||
offset_size,
|
||||
length_size,
|
||||
)?
|
||||
}
|
||||
(4, Some(4)) => {
|
||||
let spatial_chunk_dims: &[u32] = &chunk_dimensions[..rank];
|
||||
let header = ExtensibleArrayHeader::parse(
|
||||
file_data,
|
||||
addr as usize,
|
||||
offset_size,
|
||||
length_size,
|
||||
)?;
|
||||
read_extensible_array_chunks(
|
||||
file_data,
|
||||
&header,
|
||||
&dataspace.dimensions,
|
||||
spatial_chunk_dims,
|
||||
elem_size as u32,
|
||||
offset_size,
|
||||
length_size,
|
||||
)?
|
||||
}
|
||||
(v, idx) => {
|
||||
return Err(FormatError::ChunkedReadError(format!(
|
||||
"unsupported chunked layout version={v}, index_type={idx:?}"
|
||||
)));
|
||||
}
|
||||
};
|
||||
let (chunks, _) = list_chunks(
|
||||
file_data,
|
||||
layout,
|
||||
dataspace,
|
||||
elem_size,
|
||||
offset_size,
|
||||
length_size,
|
||||
)?;
|
||||
cache.populate_index(&chunks, rank);
|
||||
}
|
||||
|
||||
@@ -1211,29 +1185,12 @@ pub fn read_chunked_data_indexed(
|
||||
length_size: u8,
|
||||
cache: &ChunkCache,
|
||||
) -> Result<Vec<u8>, FormatError> {
|
||||
let (
|
||||
chunk_dimensions,
|
||||
version,
|
||||
chunk_index_type,
|
||||
addr_opt,
|
||||
single_filtered_size,
|
||||
single_filter_mask,
|
||||
) = match layout {
|
||||
let (chunk_dimensions, addr_opt) = match layout {
|
||||
DataLayout::Chunked {
|
||||
chunk_dimensions,
|
||||
btree_address,
|
||||
version,
|
||||
chunk_index_type,
|
||||
single_chunk_filtered_size,
|
||||
single_chunk_filter_mask,
|
||||
} => (
|
||||
chunk_dimensions,
|
||||
*version,
|
||||
*chunk_index_type,
|
||||
*btree_address,
|
||||
*single_chunk_filtered_size,
|
||||
*single_chunk_filter_mask,
|
||||
),
|
||||
..
|
||||
} => (chunk_dimensions, *btree_address),
|
||||
_ => {
|
||||
return Err(FormatError::ChunkedReadError(
|
||||
"expected chunked layout".into(),
|
||||
@@ -1270,69 +1227,14 @@ pub fn read_chunked_data_indexed(
|
||||
|
||||
// Build chunk index on first access
|
||||
if !cache.has_chunk_index() {
|
||||
let chunks = match (version, chunk_index_type) {
|
||||
(3, _) => collect_chunk_info(file_data, addr, ndims, offset_size, length_size)?,
|
||||
(4, Some(1)) => {
|
||||
let chunk_byte_size = checked_chunk_byte_len(&chunk_dims, elem_size)?;
|
||||
let (csize, fmask) = if let Some(fs) = single_filtered_size {
|
||||
(fs as u32, single_filter_mask.unwrap_or(0))
|
||||
} else {
|
||||
(chunk_byte_size as u32, 0)
|
||||
};
|
||||
vec![ChunkInfo {
|
||||
chunk_size: csize,
|
||||
filter_mask: fmask,
|
||||
offsets: vec![0u64; rank],
|
||||
address: addr,
|
||||
}]
|
||||
}
|
||||
(4, Some(2)) => {
|
||||
let spatial_chunk_dims: &[u32] = &chunk_dimensions[..rank];
|
||||
generate_implicit_chunks(
|
||||
addr,
|
||||
&dataspace.dimensions,
|
||||
spatial_chunk_dims,
|
||||
elem_size as u32,
|
||||
)
|
||||
}
|
||||
(4, Some(3)) => {
|
||||
let spatial_chunk_dims: &[u32] = &chunk_dimensions[..rank];
|
||||
let header =
|
||||
FixedArrayHeader::parse(file_data, addr as usize, offset_size, length_size)?;
|
||||
read_fixed_array_chunks(
|
||||
file_data,
|
||||
&header,
|
||||
&dataspace.dimensions,
|
||||
spatial_chunk_dims,
|
||||
elem_size as u32,
|
||||
offset_size,
|
||||
length_size,
|
||||
)?
|
||||
}
|
||||
(4, Some(4)) => {
|
||||
let spatial_chunk_dims: &[u32] = &chunk_dimensions[..rank];
|
||||
let header = ExtensibleArrayHeader::parse(
|
||||
file_data,
|
||||
addr as usize,
|
||||
offset_size,
|
||||
length_size,
|
||||
)?;
|
||||
read_extensible_array_chunks(
|
||||
file_data,
|
||||
&header,
|
||||
&dataspace.dimensions,
|
||||
spatial_chunk_dims,
|
||||
elem_size as u32,
|
||||
offset_size,
|
||||
length_size,
|
||||
)?
|
||||
}
|
||||
(v, idx) => {
|
||||
return Err(FormatError::ChunkedReadError(format!(
|
||||
"unsupported chunked layout version={v}, index_type={idx:?}"
|
||||
)));
|
||||
}
|
||||
};
|
||||
let (chunks, _) = list_chunks(
|
||||
file_data,
|
||||
layout,
|
||||
dataspace,
|
||||
elem_size,
|
||||
offset_size,
|
||||
length_size,
|
||||
)?;
|
||||
cache.populate_chunk_index(&chunks, rank);
|
||||
// Also populate the legacy index for compatibility
|
||||
if !cache.has_index() {
|
||||
|
||||
Reference in New Issue
Block a user