quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix(format): lay Fixed/Extensible Array chunk indexes out by max dims

Browse Source 
Both indexes place each chunk at a linear index computed from the
dataset's maximum dimensions (libhdf5's max_down_chunks), and the
Extensible Array first swizzles its unlimited dimension to the slowest
position. We linearised by the current dimensions, so any dataset whose
shape was smaller than its maxshape, or whose unlimited dimension was not
the first, read back scrambled without an error: h5py libver="latest"
files with maxshape (10, None) or (20, 10), and the libhdf5 test files
h5fc_ext*.h5 and test_ld.h5.

The linearisation now lives in chunk_grid (shared with the writers), and
slots beyond the current extent are ignored as the library does.
read_fixed_array_chunks / read_extensible_array_chunks take the
dataspace's max dimensions.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-25 21:04:19 -05:00
co-authored by Claude Opus 5.5
parent 46203ea761
commit bba1560416
6 changed files with 546 additions and 174 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/extensible_array.rs
+58 -101
View File
@@ -9,6 +9,7 @@ extern crate alloc;
#[cfg(not(feature = "std"))]
use alloc::{format, vec, vec::Vec};
use crate::chunk_grid::ChunkGrid;
use crate::chunked_read::ChunkInfo;
use crate::error::FormatError;
@@ -203,8 +204,7 @@ fn read_element(
offset_size: u8,
chunk_byte_size: u64,
linear_index: usize,
num_chunks_per_dim: &[u64],
chunk_dimensions: &[u32],
grid: &ChunkGrid,
) -> Result<(Option<ChunkInfo>, usize), FormatError> {
let os = offset_size as usize;
@@ -220,7 +220,10 @@ fn read_element(
return Ok((None, os));
}
let address = read_offset(data, pos, offset_size)?;
let offsets = index_to_chunk_offsets(linear_index, num_chunks_per_dim, chunk_dimensions);
// A slot beyond the current extent is ignored, as the library does.
let Some(offsets) = grid.offsets(linear_index as u64) else {
return Ok((None, os));
};
Ok((
Some(ChunkInfo {
chunk_size: chunk_byte_size as u32,
@@ -261,7 +264,9 @@ fn read_element(
data[fm_off + 2],
data[fm_off + 3],
]);
let offsets = index_to_chunk_offsets(linear_index, num_chunks_per_dim, chunk_dimensions);
let Some(offsets) = grid.offsets(linear_index as u64) else {
return Ok((None, elem_total));
};
Ok((
Some(ChunkInfo {
chunk_size: chunk_size as u32,
@@ -274,27 +279,6 @@ fn read_element(
}
}
/// Convert a linear chunk index to N-dimensional chunk offsets in dataset space.
fn index_to_chunk_offsets(
index: usize,
num_chunks_per_dim: &[u64],
chunk_dimensions: &[u32],
) -> Vec<u64> {
let rank = num_chunks_per_dim.len();
let mut offsets = vec![0u64; rank];
let mut remaining = index as u64;
for d in (0..rank).rev() {
let nchunks = num_chunks_per_dim[d];
if nchunks == 0 {
continue;
}
let chunk_idx = remaining % nchunks;
remaining /= nchunks;
offsets[d] = chunk_idx * chunk_dimensions[d] as u64;
}
offsets
}
/// Collect elements from a data block at the given offset.
#[allow(clippy::too_many_arguments)]
/// Layout of super block `u`, per the HDF5 spec: the number of data blocks it
@@ -339,8 +323,7 @@ fn read_data_block_elements(
offset_size: u8,
chunk_byte_size: u64,
start_index: usize,
num_chunks_per_dim: &[u64],
chunk_dimensions: &[u32],
grid: &ChunkGrid,
page_init: &[u8],
first_page: usize,
) -> Result<Vec<ChunkInfo>, FormatError> {
@@ -376,8 +359,7 @@ fn read_data_block_elements(
offset_size,
chunk_byte_size,
first_index + i,
num_chunks_per_dim,
chunk_dimensions,
grid,
)?;
if let Some(ci) = info {
chunks.push(ci);
@@ -449,25 +431,19 @@ pub fn read_extensible_array_chunks(
file_data: &[u8],
header: &ExtensibleArrayHeader,
dataset_dims: &[u64],
max_dims: Option<&[u64]>,
chunk_dimensions: &[u32],
element_size: u32,
offset_size: u8,
_length_size: u8,
) -> Result<Vec<ChunkInfo>, FormatError> {
let rank = chunk_dimensions.len();
let os = offset_size as usize;
let mut num_chunks_per_dim = Vec::with_capacity(rank);
for d in 0..rank {
let ch_dim = chunk_dimensions[d] as u64;
if ch_dim == 0 {
return Err(FormatError::ChunkedReadError(
"chunk dimension is zero".into(),
));
}
let ds_dim = dataset_dims[d];
num_chunks_per_dim.push(ds_dim.div_ceil(ch_dim));
}
// Linear indexes follow the maximum dimensions, with the unlimited
// dimension swizzled to the slowest position (see `chunk_grid`).
let dims_u64: Vec<u64> = chunk_dimensions.iter().map(|&d| d as u64).collect();
let grid = ChunkGrid::extensible_array(dataset_dims, max_dims, &dims_u64)?;
let grid = &grid;
let chunk_byte_size: u64 =
chunk_dimensions.iter().map(|&d| d as u64).product::<u64>() * element_size as u64;
@@ -557,8 +533,7 @@ pub fn read_extensible_array_chunks(
offset_size,
chunk_byte_size,
i,
&num_chunks_per_dim,
chunk_dimensions,
grid,
)?;
if let Some(ci) = info {
chunks.push(ci);
@@ -594,8 +569,7 @@ pub fn read_extensible_array_chunks(
offset_size,
chunk_byte_size,
global_index,
&num_chunks_per_dim,
chunk_dimensions,
grid,
&[],
0,
)?);
@@ -625,8 +599,7 @@ pub fn read_extensible_array_chunks(
offset_size,
chunk_byte_size,
global_index,
&num_chunks_per_dim,
chunk_dimensions,
grid,
)?);
}
global_index =
@@ -653,8 +626,7 @@ fn read_super_block(
offset_size: u8,
chunk_byte_size: u64,
start_index: usize,
num_chunks_per_dim: &[u64],
chunk_dimensions: &[u32],
grid: &ChunkGrid,
) -> Result<Vec<ChunkInfo>, FormatError> {
let os = offset_size as usize;
let sb_header_size = 4 + 1 + 1 + os + arr_off_size(header);
@@ -710,8 +682,7 @@ fn read_super_block(
offset_size,
chunk_byte_size,
global_idx,
num_chunks_per_dim,
chunk_dimensions,
grid,
bitmap,
i * npages,
)?);
@@ -735,35 +706,18 @@ mod tests {
}
#[test]
fn index_to_offsets_1d() {
let num_chunks = vec![5u64];
let chunk_dims = vec![20u32];
assert_eq!(index_to_chunk_offsets(0, &num_chunks, &chunk_dims), vec![0]);
assert_eq!(
index_to_chunk_offsets(1, &num_chunks, &chunk_dims),
vec![20]
);
assert_eq!(
index_to_chunk_offsets(4, &num_chunks, &chunk_dims),
vec![80]
);
let g = ChunkGrid::fixed_array(&[100], None, &[20]).unwrap();
assert_eq!(g.offsets(0).unwrap(), vec![0]);
assert_eq!(g.offsets(1).unwrap(), vec![20]);
assert_eq!(g.offsets(4).unwrap(), vec![80]);
}
#[test]
fn index_to_offsets_2d() {
let num_chunks = vec![3u64, 2];
let chunk_dims = vec![4u32, 3];
assert_eq!(
index_to_chunk_offsets(0, &num_chunks, &chunk_dims),
vec![0, 0]
);
assert_eq!(
index_to_chunk_offsets(1, &num_chunks, &chunk_dims),
vec![0, 3]
);
assert_eq!(
index_to_chunk_offsets(2, &num_chunks, &chunk_dims),
vec![4, 0]
);
let g = ChunkGrid::fixed_array(&[10, 6], None, &[4, 3]).unwrap();
assert_eq!(g.offsets(0).unwrap(), vec![0, 0]);
assert_eq!(g.offsets(1).unwrap(), vec![0, 3]);
assert_eq!(g.offsets(2).unwrap(), vec![4, 0]);
}
#[test]
@@ -830,7 +784,7 @@ mod tests {
index_block_address: (usize::MAX - 4) as u64,
};
let buf = vec![0u8; 64];
let r = read_extensible_array_chunks(&buf, &header, &[100], &[20], 8, 8, 8);
let r = read_extensible_array_chunks(&buf, &header, &[100], None, &[20], 8, 8, 8);
assert!(r.is_err());
}
@@ -913,9 +867,17 @@ mod tests {
let header = ExtensibleArrayHeader::parse(&file_data, aehd_offset, os, ls).unwrap();
let ds_dims = vec![40u64]; // 2 chunks × 20 elements
let chunk_dims = vec![20u32];
let chunks =
read_extensible_array_chunks(&file_data, &header, &ds_dims, &chunk_dims, 8, os, ls)
.unwrap();
let chunks = read_extensible_array_chunks(
&file_data,
&header,
&ds_dims,
None,
&chunk_dims,
8,
os,
ls,
)
.unwrap();
assert_eq!(chunks.len(), 2);
assert_eq!(chunks[0].address, base_addr);
@@ -1023,9 +985,17 @@ mod tests {
let header = ExtensibleArrayHeader::parse(&file_data, aehd_offset, os, ls).unwrap();
let ds_dims = vec![40u64];
let chunk_dims = vec![10u32];
let chunks =
read_extensible_array_chunks(&file_data, &header, &ds_dims, &chunk_dims, 8, os, ls)
.unwrap();
let chunks = read_extensible_array_chunks(
&file_data,
&header,
&ds_dims,
None,
&chunk_dims,
8,
os,
ls,
)
.unwrap();
assert_eq!(chunks.len(), 4);
for (i, c) in chunks.iter().enumerate() {
@@ -1047,10 +1017,8 @@ mod tests {
#[test]
fn read_element_unallocated() {
let data = vec![0xFFu8; 16];
let num_chunks = vec![5u64];
let chunk_dims = vec![10u32];
let (info, consumed) =
read_element(&data, 0, 0, 8, 8, 80, 0, &num_chunks, &chunk_dims).unwrap();
let grid = ChunkGrid::fixed_array(&[50], None, &[10]).unwrap();
let (info, consumed) = read_element(&data, 0, 0, 8, 8, 80, 0, &grid).unwrap();
assert!(info.is_none());
assert_eq!(consumed, 8);
}
@@ -1069,20 +1037,9 @@ mod tests {
// Filter mask
data[12..16].copy_from_slice(&0u32.to_le_bytes());
let num_chunks = vec![5u64];
let chunk_dims = vec![10u32];
let (info, consumed) = read_element(
&data,
0,
1,
elem_size as u8,
os,
80,
2,
&num_chunks,
&chunk_dims,
)
.unwrap();
let grid = ChunkGrid::fixed_array(&[50], None, &[10]).unwrap();
let (info, consumed) =
read_element(&data, 0, 1, elem_size as u8, os, 80, 2, &grid).unwrap();
let ci = info.unwrap();
assert_eq!(ci.address, 0x2000);
assert_eq!(ci.chunk_size, 120);