fix(format): lay Fixed/Extensible Array chunk indexes out by max dims
Both indexes place each chunk at a linear index computed from the dataset's maximum dimensions (libhdf5's max_down_chunks), and the Extensible Array first swizzles its unlimited dimension to the slowest position. We linearised by the current dimensions, so any dataset whose shape was smaller than its maxshape, or whose unlimited dimension was not the first, read back scrambled without an error: h5py libver="latest" files with maxshape (10, None) or (20, 10), and the libhdf5 test files h5fc_ext*.h5 and test_ld.h5. The linearisation now lives in chunk_grid (shared with the writers), and slots beyond the current extent are ignored as the library does. read_fixed_array_chunks / read_extensible_array_chunks take the dataspace's max dimensions. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
@@ -0,0 +1,201 @@
|
||||
//! Chunk-index linearisation shared by the Fixed Array and Extensible Array
|
||||
//! chunk indexes (reader and writer).
|
||||
//!
|
||||
//! Both indexes store one element per chunk at a *linear* index, and the
|
||||
//! library derives that index from the chunk's scaled coordinates
|
||||
//! (`offset / chunk_dim`) using the dataset's **maximum** dimensions, not its
|
||||
//! current ones (`H5D__farray_idx_get_addr` / `H5D__earray_idx_get_addr`,
|
||||
//! via `layout->max_down_chunks`). A dataset whose current shape is smaller
|
||||
//! than its maxshape therefore has gaps in the index, and laying it out by the
|
||||
//! current shape puts every chunk after the first row in the wrong place.
|
||||
//!
|
||||
//! The Extensible Array adds one more step: its one unlimited dimension has no
|
||||
//! finite chunk count, so the library *swizzles* the coordinates to make that
|
||||
//! dimension the slowest-varying one (`H5VM_swizzle_coords`, which moves
|
||||
//! `coords[unlim_dim]` to the front and shifts the dimensions before it right
|
||||
//! by one) before linearising with `swizzled_max_down_chunks`. When the
|
||||
//! unlimited dimension is already dimension 0 no swizzle happens.
|
||||
|
||||
#[cfg(not(feature = "std"))]
|
||||
extern crate alloc;
|
||||
|
||||
#[cfg(not(feature = "std"))]
|
||||
use alloc::{vec, vec::Vec};
|
||||
|
||||
use crate::error::FormatError;
|
||||
|
||||
/// How a chunk index maps linear element indexes to chunk coordinates.
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct ChunkGrid {
|
||||
/// Spatial chunk dimensions, in dataset order.
|
||||
chunk_dims: Vec<u64>,
|
||||
/// Chunks per dimension covering the *current* extent, in dataset order.
|
||||
cur_chunks: Vec<u64>,
|
||||
/// Dataset dimension stored at each linearisation position (slowest
|
||||
/// first). The identity except for a swizzled Extensible Array.
|
||||
order: Vec<usize>,
|
||||
/// Linear stride of each linearisation position.
|
||||
down: Vec<u64>,
|
||||
}
|
||||
|
||||
impl ChunkGrid {
|
||||
/// Grid for a Fixed Array index: row-major over the chunk counts of the
|
||||
/// maximum dimensions (`max_dims`, falling back to the current dimensions
|
||||
/// when the dataspace records none).
|
||||
pub(crate) fn fixed_array(
|
||||
cur_dims: &[u64],
|
||||
max_dims: Option<&[u64]>,
|
||||
chunk_dims: &[u64],
|
||||
) -> Result<Self, FormatError> {
|
||||
Self::build(cur_dims, max_dims, chunk_dims, None)
|
||||
}
|
||||
|
||||
/// Grid for an Extensible Array index: like the Fixed Array, but the
|
||||
/// unlimited dimension (the one whose maximum is `H5S_UNLIMITED`) is moved
|
||||
/// to the slowest-varying position first.
|
||||
pub(crate) fn extensible_array(
|
||||
cur_dims: &[u64],
|
||||
max_dims: Option<&[u64]>,
|
||||
chunk_dims: &[u64],
|
||||
) -> Result<Self, FormatError> {
|
||||
let unlim = max_dims.and_then(|m| m.iter().position(|&d| d == u64::MAX));
|
||||
Self::build(cur_dims, max_dims, chunk_dims, unlim)
|
||||
}
|
||||
|
||||
fn build(
|
||||
cur_dims: &[u64],
|
||||
max_dims: Option<&[u64]>,
|
||||
chunk_dims: &[u64],
|
||||
unlim: Option<usize>,
|
||||
) -> Result<Self, FormatError> {
|
||||
let rank = chunk_dims.len();
|
||||
if cur_dims.len() != rank || max_dims.is_some_and(|m| m.len() != rank) {
|
||||
return Err(FormatError::ChunkedReadError(
|
||||
"chunk index rank does not match the dataspace".into(),
|
||||
));
|
||||
}
|
||||
if chunk_dims.contains(&0) {
|
||||
return Err(FormatError::ChunkedReadError(
|
||||
"chunk dimension is zero".into(),
|
||||
));
|
||||
}
|
||||
let cur_chunks: Vec<u64> = cur_dims
|
||||
.iter()
|
||||
.zip(chunk_dims)
|
||||
.map(|(&d, &c)| d.div_ceil(c))
|
||||
.collect();
|
||||
// Chunk counts of the maximum extent. An unlimited dimension has no
|
||||
// finite count; it only ever sits in the slowest position, where its
|
||||
// count never enters a stride. A (corrupt) maximum smaller than the
|
||||
// current extent is widened so no allocated chunk becomes unreachable.
|
||||
let max_chunks: Vec<u64> = (0..rank)
|
||||
.map(|d| {
|
||||
let max = max_dims.map_or(cur_dims[d], |m| m[d]);
|
||||
if max == u64::MAX {
|
||||
u64::MAX
|
||||
} else {
|
||||
max.div_ceil(chunk_dims[d]).max(cur_chunks[d])
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
let mut order: Vec<usize> = (0..rank).collect();
|
||||
if let Some(u) = unlim {
|
||||
order.remove(u);
|
||||
order.insert(0, u);
|
||||
}
|
||||
let mut down = vec![1u64; rank];
|
||||
for p in (0..rank.saturating_sub(1)).rev() {
|
||||
let next = max_chunks[order[p + 1]];
|
||||
if next == u64::MAX {
|
||||
// Only reachable with more than one unlimited dimension, which
|
||||
// neither index type can describe.
|
||||
return Err(FormatError::ChunkedReadError(
|
||||
"array chunk index with more than one unlimited dimension".into(),
|
||||
));
|
||||
}
|
||||
down[p] = down[p + 1].checked_mul(next).ok_or_else(|| {
|
||||
FormatError::Overflow("chunk index linear stride overflows u64".into())
|
||||
})?;
|
||||
}
|
||||
Ok(Self {
|
||||
chunk_dims: chunk_dims.to_vec(),
|
||||
cur_chunks,
|
||||
order,
|
||||
down,
|
||||
})
|
||||
}
|
||||
|
||||
/// Dataset-space offsets of the chunk stored at linear `index`, or `None`
|
||||
/// when that chunk lies outside the current extent (the index still has a
|
||||
/// slot for it; the library ignores such chunks on read).
|
||||
pub(crate) fn offsets(&self, index: u64) -> Option<Vec<u64>> {
|
||||
let rank = self.chunk_dims.len();
|
||||
let mut offsets = vec![0u64; rank];
|
||||
let mut rem = index;
|
||||
for p in 0..rank {
|
||||
let d = self.order[p];
|
||||
let scaled = rem / self.down[p];
|
||||
rem %= self.down[p];
|
||||
if scaled >= self.cur_chunks[d] {
|
||||
return None;
|
||||
}
|
||||
offsets[d] = scaled * self.chunk_dims[d];
|
||||
}
|
||||
Some(offsets)
|
||||
}
|
||||
|
||||
/// Linear index of the chunk with scaled coordinates `scaled`
|
||||
/// (`offset / chunk_dim` per dimension, in dataset order).
|
||||
#[allow(dead_code)] // used by the writer
|
||||
pub(crate) fn linear_index(&self, scaled: &[u64]) -> u64 {
|
||||
self.order
|
||||
.iter()
|
||||
.zip(&self.down)
|
||||
.map(|(&d, &stride)| scaled[d] * stride)
|
||||
.sum()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn fixed_array_uses_max_dims() {
|
||||
// shape (4, 6), chunks (2, 3), maxshape (20, 10): 10 x 4 chunk grid.
|
||||
let g = ChunkGrid::fixed_array(&[4, 6], Some(&[20, 10]), &[2, 3]).unwrap();
|
||||
assert_eq!(g.offsets(0), Some(vec![0, 0]));
|
||||
assert_eq!(g.offsets(1), Some(vec![0, 3]));
|
||||
assert_eq!(g.offsets(2), None); // column chunk 2 is beyond the extent
|
||||
assert_eq!(g.offsets(4), Some(vec![2, 0]));
|
||||
assert_eq!(g.offsets(5), Some(vec![2, 3]));
|
||||
assert_eq!(g.offsets(8), None); // row chunk 2 is beyond the extent
|
||||
assert_eq!(g.linear_index(&[1, 1]), 5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extensible_array_swizzles_unlimited_dim() {
|
||||
// maxshape (10, None): dim 1 is unlimited and becomes slowest.
|
||||
let g = ChunkGrid::extensible_array(&[4, 6], Some(&[10, u64::MAX]), &[2, 3]).unwrap();
|
||||
// max chunks of dim 0 = 5, so index = c1 * 5 + c0.
|
||||
assert_eq!(g.linear_index(&[1, 0]), 1);
|
||||
assert_eq!(g.linear_index(&[0, 1]), 5);
|
||||
assert_eq!(g.offsets(5), Some(vec![0, 3]));
|
||||
assert_eq!(g.offsets(6), Some(vec![2, 3]));
|
||||
assert_eq!(g.offsets(2), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extensible_array_unlimited_first_is_row_major() {
|
||||
let g = ChunkGrid::extensible_array(&[4, 6], Some(&[u64::MAX, 30]), &[2, 3]).unwrap();
|
||||
// max chunks of dim 1 = 10.
|
||||
assert_eq!(g.linear_index(&[1, 1]), 11);
|
||||
assert_eq!(g.offsets(11), Some(vec![2, 3]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_two_unlimited_dims_after_the_first() {
|
||||
assert!(ChunkGrid::fixed_array(&[4, 6], Some(&[u64::MAX, u64::MAX]), &[2, 3]).is_err());
|
||||
}
|
||||
}
|
||||
@@ -593,6 +593,7 @@ pub fn list_chunks(
|
||||
file_data,
|
||||
&header,
|
||||
&dataspace.dimensions,
|
||||
dataspace.max_dimensions.as_deref(),
|
||||
spatial_chunk_dims,
|
||||
elem_size as u32,
|
||||
offset_size,
|
||||
@@ -608,6 +609,7 @@ pub fn list_chunks(
|
||||
file_data,
|
||||
&header,
|
||||
&dataspace.dimensions,
|
||||
dataspace.max_dimensions.as_deref(),
|
||||
spatial_chunk_dims,
|
||||
elem_size as u32,
|
||||
offset_size,
|
||||
|
||||
@@ -9,6 +9,7 @@ extern crate alloc;
|
||||
#[cfg(not(feature = "std"))]
|
||||
use alloc::{format, vec, vec::Vec};
|
||||
|
||||
use crate::chunk_grid::ChunkGrid;
|
||||
use crate::chunked_read::ChunkInfo;
|
||||
use crate::error::FormatError;
|
||||
|
||||
@@ -203,8 +204,7 @@ fn read_element(
|
||||
offset_size: u8,
|
||||
chunk_byte_size: u64,
|
||||
linear_index: usize,
|
||||
num_chunks_per_dim: &[u64],
|
||||
chunk_dimensions: &[u32],
|
||||
grid: &ChunkGrid,
|
||||
) -> Result<(Option<ChunkInfo>, usize), FormatError> {
|
||||
let os = offset_size as usize;
|
||||
|
||||
@@ -220,7 +220,10 @@ fn read_element(
|
||||
return Ok((None, os));
|
||||
}
|
||||
let address = read_offset(data, pos, offset_size)?;
|
||||
let offsets = index_to_chunk_offsets(linear_index, num_chunks_per_dim, chunk_dimensions);
|
||||
// A slot beyond the current extent is ignored, as the library does.
|
||||
let Some(offsets) = grid.offsets(linear_index as u64) else {
|
||||
return Ok((None, os));
|
||||
};
|
||||
Ok((
|
||||
Some(ChunkInfo {
|
||||
chunk_size: chunk_byte_size as u32,
|
||||
@@ -261,7 +264,9 @@ fn read_element(
|
||||
data[fm_off + 2],
|
||||
data[fm_off + 3],
|
||||
]);
|
||||
let offsets = index_to_chunk_offsets(linear_index, num_chunks_per_dim, chunk_dimensions);
|
||||
let Some(offsets) = grid.offsets(linear_index as u64) else {
|
||||
return Ok((None, elem_total));
|
||||
};
|
||||
Ok((
|
||||
Some(ChunkInfo {
|
||||
chunk_size: chunk_size as u32,
|
||||
@@ -274,27 +279,6 @@ fn read_element(
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert a linear chunk index to N-dimensional chunk offsets in dataset space.
|
||||
fn index_to_chunk_offsets(
|
||||
index: usize,
|
||||
num_chunks_per_dim: &[u64],
|
||||
chunk_dimensions: &[u32],
|
||||
) -> Vec<u64> {
|
||||
let rank = num_chunks_per_dim.len();
|
||||
let mut offsets = vec![0u64; rank];
|
||||
let mut remaining = index as u64;
|
||||
for d in (0..rank).rev() {
|
||||
let nchunks = num_chunks_per_dim[d];
|
||||
if nchunks == 0 {
|
||||
continue;
|
||||
}
|
||||
let chunk_idx = remaining % nchunks;
|
||||
remaining /= nchunks;
|
||||
offsets[d] = chunk_idx * chunk_dimensions[d] as u64;
|
||||
}
|
||||
offsets
|
||||
}
|
||||
|
||||
/// Collect elements from a data block at the given offset.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
/// Layout of super block `u`, per the HDF5 spec: the number of data blocks it
|
||||
@@ -339,8 +323,7 @@ fn read_data_block_elements(
|
||||
offset_size: u8,
|
||||
chunk_byte_size: u64,
|
||||
start_index: usize,
|
||||
num_chunks_per_dim: &[u64],
|
||||
chunk_dimensions: &[u32],
|
||||
grid: &ChunkGrid,
|
||||
page_init: &[u8],
|
||||
first_page: usize,
|
||||
) -> Result<Vec<ChunkInfo>, FormatError> {
|
||||
@@ -376,8 +359,7 @@ fn read_data_block_elements(
|
||||
offset_size,
|
||||
chunk_byte_size,
|
||||
first_index + i,
|
||||
num_chunks_per_dim,
|
||||
chunk_dimensions,
|
||||
grid,
|
||||
)?;
|
||||
if let Some(ci) = info {
|
||||
chunks.push(ci);
|
||||
@@ -449,25 +431,19 @@ pub fn read_extensible_array_chunks(
|
||||
file_data: &[u8],
|
||||
header: &ExtensibleArrayHeader,
|
||||
dataset_dims: &[u64],
|
||||
max_dims: Option<&[u64]>,
|
||||
chunk_dimensions: &[u32],
|
||||
element_size: u32,
|
||||
offset_size: u8,
|
||||
_length_size: u8,
|
||||
) -> Result<Vec<ChunkInfo>, FormatError> {
|
||||
let rank = chunk_dimensions.len();
|
||||
let os = offset_size as usize;
|
||||
|
||||
let mut num_chunks_per_dim = Vec::with_capacity(rank);
|
||||
for d in 0..rank {
|
||||
let ch_dim = chunk_dimensions[d] as u64;
|
||||
if ch_dim == 0 {
|
||||
return Err(FormatError::ChunkedReadError(
|
||||
"chunk dimension is zero".into(),
|
||||
));
|
||||
}
|
||||
let ds_dim = dataset_dims[d];
|
||||
num_chunks_per_dim.push(ds_dim.div_ceil(ch_dim));
|
||||
}
|
||||
// Linear indexes follow the maximum dimensions, with the unlimited
|
||||
// dimension swizzled to the slowest position (see `chunk_grid`).
|
||||
let dims_u64: Vec<u64> = chunk_dimensions.iter().map(|&d| d as u64).collect();
|
||||
let grid = ChunkGrid::extensible_array(dataset_dims, max_dims, &dims_u64)?;
|
||||
let grid = &grid;
|
||||
|
||||
let chunk_byte_size: u64 =
|
||||
chunk_dimensions.iter().map(|&d| d as u64).product::<u64>() * element_size as u64;
|
||||
@@ -557,8 +533,7 @@ pub fn read_extensible_array_chunks(
|
||||
offset_size,
|
||||
chunk_byte_size,
|
||||
i,
|
||||
&num_chunks_per_dim,
|
||||
chunk_dimensions,
|
||||
grid,
|
||||
)?;
|
||||
if let Some(ci) = info {
|
||||
chunks.push(ci);
|
||||
@@ -594,8 +569,7 @@ pub fn read_extensible_array_chunks(
|
||||
offset_size,
|
||||
chunk_byte_size,
|
||||
global_index,
|
||||
&num_chunks_per_dim,
|
||||
chunk_dimensions,
|
||||
grid,
|
||||
&[],
|
||||
0,
|
||||
)?);
|
||||
@@ -625,8 +599,7 @@ pub fn read_extensible_array_chunks(
|
||||
offset_size,
|
||||
chunk_byte_size,
|
||||
global_index,
|
||||
&num_chunks_per_dim,
|
||||
chunk_dimensions,
|
||||
grid,
|
||||
)?);
|
||||
}
|
||||
global_index =
|
||||
@@ -653,8 +626,7 @@ fn read_super_block(
|
||||
offset_size: u8,
|
||||
chunk_byte_size: u64,
|
||||
start_index: usize,
|
||||
num_chunks_per_dim: &[u64],
|
||||
chunk_dimensions: &[u32],
|
||||
grid: &ChunkGrid,
|
||||
) -> Result<Vec<ChunkInfo>, FormatError> {
|
||||
let os = offset_size as usize;
|
||||
let sb_header_size = 4 + 1 + 1 + os + arr_off_size(header);
|
||||
@@ -710,8 +682,7 @@ fn read_super_block(
|
||||
offset_size,
|
||||
chunk_byte_size,
|
||||
global_idx,
|
||||
num_chunks_per_dim,
|
||||
chunk_dimensions,
|
||||
grid,
|
||||
bitmap,
|
||||
i * npages,
|
||||
)?);
|
||||
@@ -735,35 +706,18 @@ mod tests {
|
||||
}
|
||||
#[test]
|
||||
fn index_to_offsets_1d() {
|
||||
let num_chunks = vec![5u64];
|
||||
let chunk_dims = vec![20u32];
|
||||
assert_eq!(index_to_chunk_offsets(0, &num_chunks, &chunk_dims), vec![0]);
|
||||
assert_eq!(
|
||||
index_to_chunk_offsets(1, &num_chunks, &chunk_dims),
|
||||
vec![20]
|
||||
);
|
||||
assert_eq!(
|
||||
index_to_chunk_offsets(4, &num_chunks, &chunk_dims),
|
||||
vec![80]
|
||||
);
|
||||
let g = ChunkGrid::fixed_array(&[100], None, &[20]).unwrap();
|
||||
assert_eq!(g.offsets(0).unwrap(), vec![0]);
|
||||
assert_eq!(g.offsets(1).unwrap(), vec![20]);
|
||||
assert_eq!(g.offsets(4).unwrap(), vec![80]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn index_to_offsets_2d() {
|
||||
let num_chunks = vec![3u64, 2];
|
||||
let chunk_dims = vec![4u32, 3];
|
||||
assert_eq!(
|
||||
index_to_chunk_offsets(0, &num_chunks, &chunk_dims),
|
||||
vec![0, 0]
|
||||
);
|
||||
assert_eq!(
|
||||
index_to_chunk_offsets(1, &num_chunks, &chunk_dims),
|
||||
vec![0, 3]
|
||||
);
|
||||
assert_eq!(
|
||||
index_to_chunk_offsets(2, &num_chunks, &chunk_dims),
|
||||
vec![4, 0]
|
||||
);
|
||||
let g = ChunkGrid::fixed_array(&[10, 6], None, &[4, 3]).unwrap();
|
||||
assert_eq!(g.offsets(0).unwrap(), vec![0, 0]);
|
||||
assert_eq!(g.offsets(1).unwrap(), vec![0, 3]);
|
||||
assert_eq!(g.offsets(2).unwrap(), vec![4, 0]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -830,7 +784,7 @@ mod tests {
|
||||
index_block_address: (usize::MAX - 4) as u64,
|
||||
};
|
||||
let buf = vec![0u8; 64];
|
||||
let r = read_extensible_array_chunks(&buf, &header, &[100], &[20], 8, 8, 8);
|
||||
let r = read_extensible_array_chunks(&buf, &header, &[100], None, &[20], 8, 8, 8);
|
||||
assert!(r.is_err());
|
||||
}
|
||||
|
||||
@@ -913,9 +867,17 @@ mod tests {
|
||||
let header = ExtensibleArrayHeader::parse(&file_data, aehd_offset, os, ls).unwrap();
|
||||
let ds_dims = vec![40u64]; // 2 chunks × 20 elements
|
||||
let chunk_dims = vec![20u32];
|
||||
let chunks =
|
||||
read_extensible_array_chunks(&file_data, &header, &ds_dims, &chunk_dims, 8, os, ls)
|
||||
.unwrap();
|
||||
let chunks = read_extensible_array_chunks(
|
||||
&file_data,
|
||||
&header,
|
||||
&ds_dims,
|
||||
None,
|
||||
&chunk_dims,
|
||||
8,
|
||||
os,
|
||||
ls,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(chunks.len(), 2);
|
||||
assert_eq!(chunks[0].address, base_addr);
|
||||
@@ -1023,9 +985,17 @@ mod tests {
|
||||
let header = ExtensibleArrayHeader::parse(&file_data, aehd_offset, os, ls).unwrap();
|
||||
let ds_dims = vec![40u64];
|
||||
let chunk_dims = vec![10u32];
|
||||
let chunks =
|
||||
read_extensible_array_chunks(&file_data, &header, &ds_dims, &chunk_dims, 8, os, ls)
|
||||
.unwrap();
|
||||
let chunks = read_extensible_array_chunks(
|
||||
&file_data,
|
||||
&header,
|
||||
&ds_dims,
|
||||
None,
|
||||
&chunk_dims,
|
||||
8,
|
||||
os,
|
||||
ls,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(chunks.len(), 4);
|
||||
for (i, c) in chunks.iter().enumerate() {
|
||||
@@ -1047,10 +1017,8 @@ mod tests {
|
||||
#[test]
|
||||
fn read_element_unallocated() {
|
||||
let data = vec![0xFFu8; 16];
|
||||
let num_chunks = vec![5u64];
|
||||
let chunk_dims = vec![10u32];
|
||||
let (info, consumed) =
|
||||
read_element(&data, 0, 0, 8, 8, 80, 0, &num_chunks, &chunk_dims).unwrap();
|
||||
let grid = ChunkGrid::fixed_array(&[50], None, &[10]).unwrap();
|
||||
let (info, consumed) = read_element(&data, 0, 0, 8, 8, 80, 0, &grid).unwrap();
|
||||
assert!(info.is_none());
|
||||
assert_eq!(consumed, 8);
|
||||
}
|
||||
@@ -1069,20 +1037,9 @@ mod tests {
|
||||
// Filter mask
|
||||
data[12..16].copy_from_slice(&0u32.to_le_bytes());
|
||||
|
||||
let num_chunks = vec![5u64];
|
||||
let chunk_dims = vec![10u32];
|
||||
let (info, consumed) = read_element(
|
||||
&data,
|
||||
0,
|
||||
1,
|
||||
elem_size as u8,
|
||||
os,
|
||||
80,
|
||||
2,
|
||||
&num_chunks,
|
||||
&chunk_dims,
|
||||
)
|
||||
.unwrap();
|
||||
let grid = ChunkGrid::fixed_array(&[50], None, &[10]).unwrap();
|
||||
let (info, consumed) =
|
||||
read_element(&data, 0, 1, elem_size as u8, os, 80, 2, &grid).unwrap();
|
||||
let ci = info.unwrap();
|
||||
assert_eq!(ci.address, 0x2000);
|
||||
assert_eq!(ci.chunk_size, 120);
|
||||
|
||||
@@ -6,6 +6,7 @@ extern crate alloc;
|
||||
#[cfg(not(feature = "std"))]
|
||||
use alloc::{format, vec, vec::Vec};
|
||||
|
||||
use crate::chunk_grid::ChunkGrid;
|
||||
use crate::chunked_read::ChunkInfo;
|
||||
use crate::error::FormatError;
|
||||
|
||||
@@ -151,13 +152,13 @@ pub fn read_fixed_array_chunks(
|
||||
file_data: &[u8],
|
||||
header: &FixedArrayHeader,
|
||||
dataset_dims: &[u64],
|
||||
max_dims: Option<&[u64]>,
|
||||
chunk_dimensions: &[u32],
|
||||
element_size: u32,
|
||||
offset_size: u8,
|
||||
_length_size: u8,
|
||||
) -> Result<Vec<ChunkInfo>, FormatError> {
|
||||
let db_offset = header.data_block_address as usize;
|
||||
let rank = chunk_dimensions.len();
|
||||
|
||||
// Parse data block header: FADB(4) + version(1) + client_id(1) + header_address(offset_size)
|
||||
let db_header_size = 4 + 1 + 1 + offset_size as usize;
|
||||
@@ -198,19 +199,10 @@ pub fn read_fixed_array_chunks(
|
||||
))
|
||||
};
|
||||
|
||||
// Compute chunk offsets based on index.
|
||||
// Chunks are stored in row-major order within the dataset space.
|
||||
let mut num_chunks_per_dim = Vec::with_capacity(rank);
|
||||
for d_idx in 0..rank {
|
||||
let ch_dim = chunk_dimensions[d_idx] as u64;
|
||||
if ch_dim == 0 {
|
||||
return Err(FormatError::ChunkedReadError(
|
||||
"chunk dimension is zero".into(),
|
||||
));
|
||||
}
|
||||
let ds_dim = dataset_dims[d_idx];
|
||||
num_chunks_per_dim.push(ds_dim.div_ceil(ch_dim));
|
||||
}
|
||||
// The index is laid out over the chunk grid of the *maximum* dimensions
|
||||
// (row-major), so a dataset smaller than its maxshape has gaps.
|
||||
let dims_u64: Vec<u64> = chunk_dimensions.iter().map(|&d| d as u64).collect();
|
||||
let grid = ChunkGrid::fixed_array(dataset_dims, max_dims, &dims_u64)?;
|
||||
|
||||
let chunk_byte_size: u64 =
|
||||
chunk_dimensions.iter().map(|&d| d as u64).product::<u64>() * element_size as u64;
|
||||
@@ -226,7 +218,11 @@ pub fn read_fixed_array_chunks(
|
||||
header.element_size,
|
||||
chunk_byte_size,
|
||||
)? {
|
||||
let offsets = index_to_chunk_offsets(i, &num_chunks_per_dim, chunk_dimensions);
|
||||
// A slot beyond the current extent is ignored, as the
|
||||
// library does.
|
||||
let Some(offsets) = grid.offsets(i as u64) else {
|
||||
return Ok(());
|
||||
};
|
||||
chunks.push(ChunkInfo {
|
||||
chunk_size,
|
||||
filter_mask,
|
||||
@@ -367,27 +363,6 @@ fn parse_fa_element(
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert a linear chunk index to N-dimensional chunk offsets in dataset space.
|
||||
fn index_to_chunk_offsets(
|
||||
index: usize,
|
||||
num_chunks_per_dim: &[u64],
|
||||
chunk_dimensions: &[u32],
|
||||
) -> Vec<u64> {
|
||||
let rank = num_chunks_per_dim.len();
|
||||
let mut offsets = vec![0u64; rank];
|
||||
let mut remaining = index as u64;
|
||||
for d in (0..rank).rev() {
|
||||
let nchunks = num_chunks_per_dim[d];
|
||||
if nchunks == 0 {
|
||||
continue;
|
||||
}
|
||||
let chunk_idx = remaining % nchunks;
|
||||
remaining /= nchunks;
|
||||
offsets[d] = chunk_idx * chunk_dimensions[d] as u64;
|
||||
}
|
||||
offsets
|
||||
}
|
||||
|
||||
/// Read a variable-length little-endian unsigned integer.
|
||||
fn read_variable_length(data: &[u8], size: usize) -> Result<u64, FormatError> {
|
||||
if size > 8 || data.len() < size {
|
||||
@@ -416,44 +391,21 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn index_to_offsets_1d() {
|
||||
let num_chunks = vec![5u64];
|
||||
let chunk_dims = vec![20u32];
|
||||
assert_eq!(index_to_chunk_offsets(0, &num_chunks, &chunk_dims), vec![0]);
|
||||
assert_eq!(
|
||||
index_to_chunk_offsets(1, &num_chunks, &chunk_dims),
|
||||
vec![20]
|
||||
);
|
||||
assert_eq!(
|
||||
index_to_chunk_offsets(4, &num_chunks, &chunk_dims),
|
||||
vec![80]
|
||||
);
|
||||
let g = ChunkGrid::fixed_array(&[100], None, &[20]).unwrap();
|
||||
assert_eq!(g.offsets(0).unwrap(), vec![0]);
|
||||
assert_eq!(g.offsets(1).unwrap(), vec![20]);
|
||||
assert_eq!(g.offsets(4).unwrap(), vec![80]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn index_to_offsets_2d() {
|
||||
// 10x6 dataset with 4x3 chunks => ceil(10/4)=3, ceil(6/3)=2 => 6 chunks
|
||||
let num_chunks = vec![3u64, 2];
|
||||
let chunk_dims = vec![4u32, 3];
|
||||
assert_eq!(
|
||||
index_to_chunk_offsets(0, &num_chunks, &chunk_dims),
|
||||
vec![0, 0]
|
||||
);
|
||||
assert_eq!(
|
||||
index_to_chunk_offsets(1, &num_chunks, &chunk_dims),
|
||||
vec![0, 3]
|
||||
);
|
||||
assert_eq!(
|
||||
index_to_chunk_offsets(2, &num_chunks, &chunk_dims),
|
||||
vec![4, 0]
|
||||
);
|
||||
assert_eq!(
|
||||
index_to_chunk_offsets(3, &num_chunks, &chunk_dims),
|
||||
vec![4, 3]
|
||||
);
|
||||
assert_eq!(
|
||||
index_to_chunk_offsets(5, &num_chunks, &chunk_dims),
|
||||
vec![8, 3]
|
||||
);
|
||||
let g = ChunkGrid::fixed_array(&[10, 6], None, &[4, 3]).unwrap();
|
||||
assert_eq!(g.offsets(0).unwrap(), vec![0, 0]);
|
||||
assert_eq!(g.offsets(1).unwrap(), vec![0, 3]);
|
||||
assert_eq!(g.offsets(2).unwrap(), vec![4, 0]);
|
||||
assert_eq!(g.offsets(3).unwrap(), vec![4, 3]);
|
||||
assert_eq!(g.offsets(5).unwrap(), vec![8, 3]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -517,7 +469,7 @@ mod tests {
|
||||
|
||||
let read = |f: &[u8], fahd: usize| -> Result<Vec<ChunkInfo>, FormatError> {
|
||||
let h = FixedArrayHeader::parse(f, fahd, 8, 8)?;
|
||||
read_fixed_array_chunks(f, &h, &[60], &[20], 8, 8, 8)
|
||||
read_fixed_array_chunks(f, &h, &[60], None, &[20], 8, 8, 8)
|
||||
};
|
||||
|
||||
let (clean, fahd) = build();
|
||||
@@ -562,7 +514,7 @@ mod tests {
|
||||
let db = 0x100usize;
|
||||
buf[db..db + 4].copy_from_slice(b"FADB");
|
||||
let header = FixedArrayHeader::parse(&buf, fahd, 8, 8).unwrap();
|
||||
let r = read_fixed_array_chunks(&buf, &header, &[100], &[20], 8, 8, 8);
|
||||
let r = read_fixed_array_chunks(&buf, &header, &[100], None, &[20], 8, 8, 8);
|
||||
assert!(r.is_err());
|
||||
}
|
||||
|
||||
@@ -579,7 +531,7 @@ mod tests {
|
||||
stamp_checksum(&mut buf, fahd, fahd + 24);
|
||||
buf[0x80..0x84].copy_from_slice(b"FADB");
|
||||
let header = FixedArrayHeader::parse(&buf, fahd, 8, 8).unwrap();
|
||||
let r = read_fixed_array_chunks(&buf, &header, &[100], &[20], 8, 8, 8);
|
||||
let r = read_fixed_array_chunks(&buf, &header, &[100], None, &[20], 8, 8, 8);
|
||||
assert!(r.is_err());
|
||||
}
|
||||
|
||||
@@ -602,7 +554,7 @@ mod tests {
|
||||
data_block_address: (usize::MAX - 4) as u64,
|
||||
};
|
||||
let buf = vec![0u8; 64];
|
||||
let r = read_fixed_array_chunks(&buf, &header, &[100], &[20], 8, 8, 8);
|
||||
let r = read_fixed_array_chunks(&buf, &header, &[100], None, &[20], 8, 8, 8);
|
||||
assert!(r.is_err());
|
||||
}
|
||||
|
||||
@@ -664,6 +616,7 @@ mod tests {
|
||||
&file_data,
|
||||
&header,
|
||||
&ds_dims,
|
||||
None,
|
||||
&chunk_dims,
|
||||
8,
|
||||
offset_size,
|
||||
@@ -740,6 +693,7 @@ mod tests {
|
||||
&file_data,
|
||||
&header,
|
||||
&ds_dims,
|
||||
None,
|
||||
&chunk_dims,
|
||||
8,
|
||||
offset_size,
|
||||
@@ -840,6 +794,7 @@ mod tests {
|
||||
&file_data,
|
||||
&header,
|
||||
&ds_dims,
|
||||
None,
|
||||
&chunk_dims,
|
||||
8,
|
||||
offset_size,
|
||||
|
||||
@@ -54,6 +54,7 @@ pub mod btree_v1;
|
||||
pub mod btree_v2;
|
||||
pub mod checksum;
|
||||
pub mod chunk_cache;
|
||||
mod chunk_grid;
|
||||
pub mod chunk_index;
|
||||
pub mod chunked_read;
|
||||
pub mod chunked_write;
|
||||
|
||||
@@ -0,0 +1,256 @@
|
||||
//! Fixed Array / Extensible Array chunk-index interop with libhdf5 (via h5py).
|
||||
//!
|
||||
//! Both indexes place each chunk at a linear index computed from the
|
||||
//! dataset's *maximum* dimensions, and the Extensible Array additionally
|
||||
//! moves its unlimited dimension to the slowest-varying position. Getting
|
||||
//! either wrong reads (or writes) every chunk after the first row in the
|
||||
//! wrong place, silently, so these tests compare every value.
|
||||
//!
|
||||
//! Skipped when python3 with h5py is unavailable, unless
|
||||
//! `CLAWHDF5_REQUIRE_INTEROP=1`.
|
||||
|
||||
use std::process::Command;
|
||||
|
||||
use clawhdf5::File;
|
||||
|
||||
fn python() -> String {
|
||||
std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
|
||||
}
|
||||
|
||||
fn interop_required() -> bool {
|
||||
std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
|
||||
}
|
||||
|
||||
fn python_available() -> bool {
|
||||
Command::new(python())
|
||||
.args(["-c", "import h5py"])
|
||||
.output()
|
||||
.map(|o| o.status.success())
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
macro_rules! skip_if_no_python {
|
||||
() => {
|
||||
if !python_available() {
|
||||
assert!(
|
||||
!interop_required(),
|
||||
"CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available"
|
||||
);
|
||||
eprintln!("SKIP: python3 with h5py not available");
|
||||
return;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
fn run_python(script: &str) -> String {
|
||||
let output = Command::new(python())
|
||||
.args(["-c", script])
|
||||
.output()
|
||||
.expect("failed to run python");
|
||||
if !output.status.success() {
|
||||
panic!(
|
||||
"Python script failed:\nSTDOUT: {}\nSTDERR: {}",
|
||||
String::from_utf8_lossy(&output.stdout),
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
}
|
||||
String::from_utf8_lossy(&output.stdout).trim().to_string()
|
||||
}
|
||||
|
||||
/// Row-major `arange` of `shape`, cropped to `crop` (the current extent).
|
||||
fn arange_cropped(full: &[usize], crop: &[usize]) -> Vec<i32> {
|
||||
let n: usize = crop.iter().product();
|
||||
let mut out = Vec::with_capacity(n);
|
||||
for flat in 0..n {
|
||||
let mut rem = flat;
|
||||
let mut src = 0usize;
|
||||
let mut stride = 1usize;
|
||||
let mut coords = vec![0usize; crop.len()];
|
||||
for d in (0..crop.len()).rev() {
|
||||
coords[d] = rem % crop[d];
|
||||
rem /= crop[d];
|
||||
}
|
||||
for d in (0..full.len()).rev() {
|
||||
src += coords[d] * stride;
|
||||
stride *= full[d];
|
||||
}
|
||||
out.push(src as i32);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// One `i4` dataset, filled with `arange` over `full` and then resized to
|
||||
/// `shape` (equal to `full` unless the case shrinks it).
|
||||
struct Case {
|
||||
name: &'static str,
|
||||
full: Vec<usize>,
|
||||
shape: Vec<usize>,
|
||||
chunks: Vec<usize>,
|
||||
maxshape: &'static str,
|
||||
extra: &'static str,
|
||||
index: &'static str,
|
||||
}
|
||||
|
||||
fn py_tuple(v: &[usize]) -> String {
|
||||
let parts: Vec<String> = v.iter().map(|x| x.to_string()).collect();
|
||||
format!("({},)", parts.join(","))
|
||||
}
|
||||
|
||||
/// Have h5py (`libver="latest"`, so Fixed/Extensible Array indexes) write
|
||||
/// every case to one file, then read each back and compare every value.
|
||||
fn check_h5py_written(cases: &[Case]) {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("h5py_chunk_index.h5");
|
||||
let path_str = path.display().to_string();
|
||||
|
||||
let mut script =
|
||||
format!("import h5py, numpy as np\nf = h5py.File(r'{path_str}', 'w', libver='latest')\n");
|
||||
for c in cases {
|
||||
script += &format!(
|
||||
"d = f.create_dataset('{name}', data=np.arange({n}, dtype='i4').reshape({full}), \
|
||||
chunks={chunks}, maxshape={maxshape}{extra})\n\
|
||||
d.resize({shape})\n",
|
||||
name = c.name,
|
||||
n = c.full.iter().product::<usize>(),
|
||||
full = py_tuple(&c.full),
|
||||
chunks = py_tuple(&c.chunks),
|
||||
maxshape = c.maxshape,
|
||||
extra = c.extra,
|
||||
shape = py_tuple(&c.shape),
|
||||
);
|
||||
}
|
||||
script += "f.close()\n";
|
||||
run_python(&script);
|
||||
|
||||
let file = File::open(&path).unwrap();
|
||||
for c in cases {
|
||||
let ds = file.dataset(c.name).unwrap();
|
||||
let shape: Vec<usize> = ds.shape().unwrap().iter().map(|&d| d as usize).collect();
|
||||
assert_eq!(shape, c.shape, "{}: shape", c.name);
|
||||
let got = ds.read_i32().unwrap();
|
||||
let want = arange_cropped(&c.full, &c.shape);
|
||||
let bad = got.iter().zip(&want).filter(|(a, b)| a != b).count();
|
||||
assert_eq!(
|
||||
got,
|
||||
want,
|
||||
"{}: {bad} of {} values differ (index {})",
|
||||
c.name,
|
||||
want.len(),
|
||||
c.index
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// h5py-written Extensible Array whose unlimited dimension is not the first,
|
||||
/// with the current shape smaller than the finite maximum: the library
|
||||
/// swizzles the unlimited dimension to the slowest position and strides the
|
||||
/// rest by their maximum chunk counts.
|
||||
#[test]
|
||||
fn h5py_extensible_array_partial_extent_reads_correctly() {
|
||||
skip_if_no_python!();
|
||||
check_h5py_written(&[
|
||||
// The `ea_fa_partial.h5` repro from the conformance sweep.
|
||||
Case {
|
||||
name: "ea_10_none",
|
||||
full: vec![4, 6],
|
||||
shape: vec![4, 6],
|
||||
chunks: vec![2, 3],
|
||||
maxshape: "(10, None)",
|
||||
extra: "",
|
||||
index: "EA, unlimited dim 1",
|
||||
},
|
||||
Case {
|
||||
name: "ea_none_10",
|
||||
full: vec![4, 6],
|
||||
shape: vec![4, 6],
|
||||
chunks: vec![2, 3],
|
||||
maxshape: "(None, 10)",
|
||||
extra: "",
|
||||
index: "EA, unlimited dim 0",
|
||||
},
|
||||
Case {
|
||||
name: "ea_3d_mid",
|
||||
full: vec![3, 4, 5],
|
||||
shape: vec![3, 4, 5],
|
||||
chunks: vec![2, 3, 2],
|
||||
maxshape: "(5, None, 7)",
|
||||
extra: "",
|
||||
index: "EA, unlimited dim 1 of 3",
|
||||
},
|
||||
Case {
|
||||
name: "ea_3d_last_gzip",
|
||||
full: vec![3, 4, 5],
|
||||
shape: vec![3, 4, 5],
|
||||
chunks: vec![2, 3, 2],
|
||||
maxshape: "(5, 9, None)",
|
||||
extra: ", compression='gzip'",
|
||||
index: "EA, unlimited dim 2 of 3, filtered",
|
||||
},
|
||||
// Many chunks: crosses data blocks, super blocks and paging.
|
||||
Case {
|
||||
name: "ea_many",
|
||||
full: vec![3, 1500],
|
||||
shape: vec![3, 1500],
|
||||
chunks: vec![1, 1],
|
||||
maxshape: "(4, None)",
|
||||
extra: "",
|
||||
index: "EA, 4500 slots",
|
||||
},
|
||||
// Shrunk after writing: chunks beyond the extent must be ignored.
|
||||
Case {
|
||||
name: "ea_shrunk",
|
||||
full: vec![8, 9],
|
||||
shape: vec![3, 4],
|
||||
chunks: vec![2, 3],
|
||||
maxshape: "(10, None)",
|
||||
extra: "",
|
||||
index: "EA, shrunk",
|
||||
},
|
||||
]);
|
||||
}
|
||||
|
||||
/// h5py-written Fixed Array with the current shape smaller than a finite
|
||||
/// maxshape: the index has one slot per chunk of the *maximum* extent.
|
||||
#[test]
|
||||
fn h5py_fixed_array_partial_extent_reads_correctly() {
|
||||
skip_if_no_python!();
|
||||
check_h5py_written(&[
|
||||
Case {
|
||||
name: "fa_20_10",
|
||||
full: vec![4, 6],
|
||||
shape: vec![4, 6],
|
||||
chunks: vec![2, 3],
|
||||
maxshape: "(20, 10)",
|
||||
extra: "",
|
||||
index: "FA",
|
||||
},
|
||||
Case {
|
||||
name: "fa_3d_gzip",
|
||||
full: vec![3, 4, 5],
|
||||
shape: vec![3, 4, 5],
|
||||
chunks: vec![2, 3, 2],
|
||||
maxshape: "(6, 8, 10)",
|
||||
extra: ", compression='gzip'",
|
||||
index: "FA, filtered",
|
||||
},
|
||||
// Paged (> 1024 slots) with most of them beyond the extent.
|
||||
Case {
|
||||
name: "fa_paged",
|
||||
full: vec![30, 50],
|
||||
shape: vec![30, 50],
|
||||
chunks: vec![1, 1],
|
||||
maxshape: "(40, 60)",
|
||||
extra: "",
|
||||
index: "FA, 2400 slots, paged",
|
||||
},
|
||||
Case {
|
||||
name: "fa_shrunk",
|
||||
full: vec![8, 9],
|
||||
shape: vec![5, 2],
|
||||
chunks: vec![2, 3],
|
||||
maxshape: "(20, 10)",
|
||||
extra: "",
|
||||
index: "FA, shrunk",
|
||||
},
|
||||
]);
|
||||
}
|
||||
Reference in New Issue
Block a user