fix(format): lay Fixed/Extensible Array chunk indexes out by max dims
Both indexes place each chunk at a linear index computed from the dataset's maximum dimensions (libhdf5's max_down_chunks), and the Extensible Array first swizzles its unlimited dimension to the slowest position. We linearised by the current dimensions, so any dataset whose shape was smaller than its maxshape, or whose unlimited dimension was not the first, read back scrambled without an error: h5py libver="latest" files with maxshape (10, None) or (20, 10), and the libhdf5 test files h5fc_ext*.h5 and test_ld.h5. The linearisation now lives in chunk_grid (shared with the writers), and slots beyond the current extent are ignored as the library does. read_fixed_array_chunks / read_extensible_array_chunks take the dataspace's max dimensions. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
@@ -6,6 +6,7 @@ extern crate alloc;
|
||||
#[cfg(not(feature = "std"))]
|
||||
use alloc::{format, vec, vec::Vec};
|
||||
|
||||
use crate::chunk_grid::ChunkGrid;
|
||||
use crate::chunked_read::ChunkInfo;
|
||||
use crate::error::FormatError;
|
||||
|
||||
@@ -151,13 +152,13 @@ pub fn read_fixed_array_chunks(
|
||||
file_data: &[u8],
|
||||
header: &FixedArrayHeader,
|
||||
dataset_dims: &[u64],
|
||||
max_dims: Option<&[u64]>,
|
||||
chunk_dimensions: &[u32],
|
||||
element_size: u32,
|
||||
offset_size: u8,
|
||||
_length_size: u8,
|
||||
) -> Result<Vec<ChunkInfo>, FormatError> {
|
||||
let db_offset = header.data_block_address as usize;
|
||||
let rank = chunk_dimensions.len();
|
||||
|
||||
// Parse data block header: FADB(4) + version(1) + client_id(1) + header_address(offset_size)
|
||||
let db_header_size = 4 + 1 + 1 + offset_size as usize;
|
||||
@@ -198,19 +199,10 @@ pub fn read_fixed_array_chunks(
|
||||
))
|
||||
};
|
||||
|
||||
// Compute chunk offsets based on index.
|
||||
// Chunks are stored in row-major order within the dataset space.
|
||||
let mut num_chunks_per_dim = Vec::with_capacity(rank);
|
||||
for d_idx in 0..rank {
|
||||
let ch_dim = chunk_dimensions[d_idx] as u64;
|
||||
if ch_dim == 0 {
|
||||
return Err(FormatError::ChunkedReadError(
|
||||
"chunk dimension is zero".into(),
|
||||
));
|
||||
}
|
||||
let ds_dim = dataset_dims[d_idx];
|
||||
num_chunks_per_dim.push(ds_dim.div_ceil(ch_dim));
|
||||
}
|
||||
// The index is laid out over the chunk grid of the *maximum* dimensions
|
||||
// (row-major), so a dataset smaller than its maxshape has gaps.
|
||||
let dims_u64: Vec<u64> = chunk_dimensions.iter().map(|&d| d as u64).collect();
|
||||
let grid = ChunkGrid::fixed_array(dataset_dims, max_dims, &dims_u64)?;
|
||||
|
||||
let chunk_byte_size: u64 =
|
||||
chunk_dimensions.iter().map(|&d| d as u64).product::<u64>() * element_size as u64;
|
||||
@@ -226,7 +218,11 @@ pub fn read_fixed_array_chunks(
|
||||
header.element_size,
|
||||
chunk_byte_size,
|
||||
)? {
|
||||
let offsets = index_to_chunk_offsets(i, &num_chunks_per_dim, chunk_dimensions);
|
||||
// A slot beyond the current extent is ignored, as the
|
||||
// library does.
|
||||
let Some(offsets) = grid.offsets(i as u64) else {
|
||||
return Ok(());
|
||||
};
|
||||
chunks.push(ChunkInfo {
|
||||
chunk_size,
|
||||
filter_mask,
|
||||
@@ -367,27 +363,6 @@ fn parse_fa_element(
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert a linear chunk index to N-dimensional chunk offsets in dataset space.
|
||||
fn index_to_chunk_offsets(
|
||||
index: usize,
|
||||
num_chunks_per_dim: &[u64],
|
||||
chunk_dimensions: &[u32],
|
||||
) -> Vec<u64> {
|
||||
let rank = num_chunks_per_dim.len();
|
||||
let mut offsets = vec![0u64; rank];
|
||||
let mut remaining = index as u64;
|
||||
for d in (0..rank).rev() {
|
||||
let nchunks = num_chunks_per_dim[d];
|
||||
if nchunks == 0 {
|
||||
continue;
|
||||
}
|
||||
let chunk_idx = remaining % nchunks;
|
||||
remaining /= nchunks;
|
||||
offsets[d] = chunk_idx * chunk_dimensions[d] as u64;
|
||||
}
|
||||
offsets
|
||||
}
|
||||
|
||||
/// Read a variable-length little-endian unsigned integer.
|
||||
fn read_variable_length(data: &[u8], size: usize) -> Result<u64, FormatError> {
|
||||
if size > 8 || data.len() < size {
|
||||
@@ -416,44 +391,21 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn index_to_offsets_1d() {
|
||||
let num_chunks = vec![5u64];
|
||||
let chunk_dims = vec![20u32];
|
||||
assert_eq!(index_to_chunk_offsets(0, &num_chunks, &chunk_dims), vec![0]);
|
||||
assert_eq!(
|
||||
index_to_chunk_offsets(1, &num_chunks, &chunk_dims),
|
||||
vec![20]
|
||||
);
|
||||
assert_eq!(
|
||||
index_to_chunk_offsets(4, &num_chunks, &chunk_dims),
|
||||
vec![80]
|
||||
);
|
||||
let g = ChunkGrid::fixed_array(&[100], None, &[20]).unwrap();
|
||||
assert_eq!(g.offsets(0).unwrap(), vec![0]);
|
||||
assert_eq!(g.offsets(1).unwrap(), vec![20]);
|
||||
assert_eq!(g.offsets(4).unwrap(), vec![80]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn index_to_offsets_2d() {
|
||||
// 10x6 dataset with 4x3 chunks => ceil(10/4)=3, ceil(6/3)=2 => 6 chunks
|
||||
let num_chunks = vec![3u64, 2];
|
||||
let chunk_dims = vec![4u32, 3];
|
||||
assert_eq!(
|
||||
index_to_chunk_offsets(0, &num_chunks, &chunk_dims),
|
||||
vec![0, 0]
|
||||
);
|
||||
assert_eq!(
|
||||
index_to_chunk_offsets(1, &num_chunks, &chunk_dims),
|
||||
vec![0, 3]
|
||||
);
|
||||
assert_eq!(
|
||||
index_to_chunk_offsets(2, &num_chunks, &chunk_dims),
|
||||
vec![4, 0]
|
||||
);
|
||||
assert_eq!(
|
||||
index_to_chunk_offsets(3, &num_chunks, &chunk_dims),
|
||||
vec![4, 3]
|
||||
);
|
||||
assert_eq!(
|
||||
index_to_chunk_offsets(5, &num_chunks, &chunk_dims),
|
||||
vec![8, 3]
|
||||
);
|
||||
let g = ChunkGrid::fixed_array(&[10, 6], None, &[4, 3]).unwrap();
|
||||
assert_eq!(g.offsets(0).unwrap(), vec![0, 0]);
|
||||
assert_eq!(g.offsets(1).unwrap(), vec![0, 3]);
|
||||
assert_eq!(g.offsets(2).unwrap(), vec![4, 0]);
|
||||
assert_eq!(g.offsets(3).unwrap(), vec![4, 3]);
|
||||
assert_eq!(g.offsets(5).unwrap(), vec![8, 3]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -517,7 +469,7 @@ mod tests {
|
||||
|
||||
let read = |f: &[u8], fahd: usize| -> Result<Vec<ChunkInfo>, FormatError> {
|
||||
let h = FixedArrayHeader::parse(f, fahd, 8, 8)?;
|
||||
read_fixed_array_chunks(f, &h, &[60], &[20], 8, 8, 8)
|
||||
read_fixed_array_chunks(f, &h, &[60], None, &[20], 8, 8, 8)
|
||||
};
|
||||
|
||||
let (clean, fahd) = build();
|
||||
@@ -562,7 +514,7 @@ mod tests {
|
||||
let db = 0x100usize;
|
||||
buf[db..db + 4].copy_from_slice(b"FADB");
|
||||
let header = FixedArrayHeader::parse(&buf, fahd, 8, 8).unwrap();
|
||||
let r = read_fixed_array_chunks(&buf, &header, &[100], &[20], 8, 8, 8);
|
||||
let r = read_fixed_array_chunks(&buf, &header, &[100], None, &[20], 8, 8, 8);
|
||||
assert!(r.is_err());
|
||||
}
|
||||
|
||||
@@ -579,7 +531,7 @@ mod tests {
|
||||
stamp_checksum(&mut buf, fahd, fahd + 24);
|
||||
buf[0x80..0x84].copy_from_slice(b"FADB");
|
||||
let header = FixedArrayHeader::parse(&buf, fahd, 8, 8).unwrap();
|
||||
let r = read_fixed_array_chunks(&buf, &header, &[100], &[20], 8, 8, 8);
|
||||
let r = read_fixed_array_chunks(&buf, &header, &[100], None, &[20], 8, 8, 8);
|
||||
assert!(r.is_err());
|
||||
}
|
||||
|
||||
@@ -602,7 +554,7 @@ mod tests {
|
||||
data_block_address: (usize::MAX - 4) as u64,
|
||||
};
|
||||
let buf = vec![0u8; 64];
|
||||
let r = read_fixed_array_chunks(&buf, &header, &[100], &[20], 8, 8, 8);
|
||||
let r = read_fixed_array_chunks(&buf, &header, &[100], None, &[20], 8, 8, 8);
|
||||
assert!(r.is_err());
|
||||
}
|
||||
|
||||
@@ -664,6 +616,7 @@ mod tests {
|
||||
&file_data,
|
||||
&header,
|
||||
&ds_dims,
|
||||
None,
|
||||
&chunk_dims,
|
||||
8,
|
||||
offset_size,
|
||||
@@ -740,6 +693,7 @@ mod tests {
|
||||
&file_data,
|
||||
&header,
|
||||
&ds_dims,
|
||||
None,
|
||||
&chunk_dims,
|
||||
8,
|
||||
offset_size,
|
||||
@@ -840,6 +794,7 @@ mod tests {
|
||||
&file_data,
|
||||
&header,
|
||||
&ds_dims,
|
||||
None,
|
||||
&chunk_dims,
|
||||
8,
|
||||
offset_size,
|
||||
|
||||
Reference in New Issue
Block a user