diff --git a/crates/clawhdf5-format/src/chunked_read.rs b/crates/clawhdf5-format/src/chunked_read.rs index e249168..f428285 100644 --- a/crates/clawhdf5-format/src/chunked_read.rs +++ b/crates/clawhdf5-format/src/chunked_read.rs @@ -933,16 +933,40 @@ pub fn generate_implicit_chunks( dataset_dims: &[u64], chunk_dimensions: &[u32], element_size: u32, +) -> Vec { + generate_implicit_chunks_in_grid( + base_address, + dataset_dims, + dataset_dims, + chunk_dimensions, + element_size, + ) +} + +/// [`generate_implicit_chunks`] for a dataset whose maximum dimensions +/// (`max_dims`) exceed its current ones: libhdf5 allocates the chunks of +/// the whole maximum extent and places chunk `scaled` at its row-major +/// position in the *maximum* chunk grid (`H5D__none_idx_get_addr`, +/// `max_down_chunks`), so the current extent's chunks are not contiguous. +/// Only the chunks of the current extent are listed. +pub fn generate_implicit_chunks_in_grid( + base_address: u64, + dataset_dims: &[u64], + max_dims: &[u64], + chunk_dimensions: &[u32], + element_size: u32, ) -> Vec { let rank = chunk_dimensions.len(); let chunk_byte_size: u64 = chunk_dimensions.iter().map(|&d| d as u64).product::() * element_size as u64; let mut num_chunks_per_dim = Vec::with_capacity(rank); + let mut grid_per_dim = Vec::with_capacity(rank); for d in 0..rank { - let ds = dataset_dims[d]; let ch = chunk_dimensions[d] as u64; - num_chunks_per_dim.push(ds.div_ceil(ch)); + let n = dataset_dims[d].div_ceil(ch); + num_chunks_per_dim.push(n); + grid_per_dim.push(max_dims.get(d).map_or(n, |m| m.div_ceil(ch)).max(n)); } let total_chunks: u64 = num_chunks_per_dim.iter().product(); @@ -951,18 +975,22 @@ pub fn generate_implicit_chunks( for linear_idx in 0..total_chunks { let mut offsets = vec![0u64; rank]; let mut remaining = linear_idx; + let mut grid_idx = 0u64; + let mut down = 1u64; for d in (0..rank).rev() { let nchunks = num_chunks_per_dim[d]; let chunk_idx = remaining % nchunks; remaining /= nchunks; offsets[d] = chunk_idx * chunk_dimensions[d] as u64; + grid_idx = grid_idx.saturating_add(chunk_idx.saturating_mul(down)); + down = down.saturating_mul(grid_per_dim[d]); } chunks.push(ChunkInfo { chunk_size: chunk_byte_size as u32, filter_mask: 0, offsets, - address: base_address + linear_idx * chunk_byte_size, + address: base_address.saturating_add(grid_idx.saturating_mul(chunk_byte_size)), }); } @@ -1153,9 +1181,13 @@ pub fn list_chunks( (4, Some(2)) => { // Implicit index — use spatial chunk dims only let spatial_chunk_dims: &[u32] = &chunk_dimensions[..rank]; - generate_implicit_chunks( + generate_implicit_chunks_in_grid( addr, &dataspace.dimensions, + dataspace + .max_dimensions + .as_deref() + .unwrap_or(&dataspace.dimensions), spatial_chunk_dims, elem_size as u32, ) @@ -3015,6 +3047,17 @@ mod tests { } } + /// A dataset below its maximum extent: libhdf5 lays chunks out over the + /// maximum chunk grid, so row 1 starts after a whole maximum row (here + /// 4 chunks), not after the current row of 3. + #[test] + fn implicit_chunks_use_the_maximum_grid() { + let chunks = generate_implicit_chunks_in_grid(0x100, &[2, 3], &[4, 4], &[1, 1], 4); + let addrs: Vec = chunks.iter().map(|c| (c.address - 0x100) / 4).collect(); + assert_eq!(addrs, vec![0, 1, 2, 4, 5, 6]); + assert_eq!(chunks[3].offsets, vec![1, 0]); + } + #[test] fn implicit_chunks_partial_last() { // 25 elements, chunk size 10 => 3 chunks (last partial)