diff --git a/crates/clawhdf5-format/src/chunk_grid.rs b/crates/clawhdf5-format/src/chunk_grid.rs new file mode 100644 index 0000000..c5a06fd --- /dev/null +++ b/crates/clawhdf5-format/src/chunk_grid.rs @@ -0,0 +1,201 @@ +//! Chunk-index linearisation shared by the Fixed Array and Extensible Array +//! chunk indexes (reader and writer). +//! +//! Both indexes store one element per chunk at a *linear* index, and the +//! library derives that index from the chunk's scaled coordinates +//! (`offset / chunk_dim`) using the dataset's **maximum** dimensions, not its +//! current ones (`H5D__farray_idx_get_addr` / `H5D__earray_idx_get_addr`, +//! via `layout->max_down_chunks`). A dataset whose current shape is smaller +//! than its maxshape therefore has gaps in the index, and laying it out by the +//! current shape puts every chunk after the first row in the wrong place. +//! +//! The Extensible Array adds one more step: its one unlimited dimension has no +//! finite chunk count, so the library *swizzles* the coordinates to make that +//! dimension the slowest-varying one (`H5VM_swizzle_coords`, which moves +//! `coords[unlim_dim]` to the front and shifts the dimensions before it right +//! by one) before linearising with `swizzled_max_down_chunks`. When the +//! unlimited dimension is already dimension 0 no swizzle happens. + +#[cfg(not(feature = "std"))] +extern crate alloc; + +#[cfg(not(feature = "std"))] +use alloc::{vec, vec::Vec}; + +use crate::error::FormatError; + +/// How a chunk index maps linear element indexes to chunk coordinates. +#[derive(Debug, Clone)] +pub(crate) struct ChunkGrid { + /// Spatial chunk dimensions, in dataset order. + chunk_dims: Vec, + /// Chunks per dimension covering the *current* extent, in dataset order. + cur_chunks: Vec, + /// Dataset dimension stored at each linearisation position (slowest + /// first). The identity except for a swizzled Extensible Array. + order: Vec, + /// Linear stride of each linearisation position. + down: Vec, +} + +impl ChunkGrid { + /// Grid for a Fixed Array index: row-major over the chunk counts of the + /// maximum dimensions (`max_dims`, falling back to the current dimensions + /// when the dataspace records none). + pub(crate) fn fixed_array( + cur_dims: &[u64], + max_dims: Option<&[u64]>, + chunk_dims: &[u64], + ) -> Result { + Self::build(cur_dims, max_dims, chunk_dims, None) + } + + /// Grid for an Extensible Array index: like the Fixed Array, but the + /// unlimited dimension (the one whose maximum is `H5S_UNLIMITED`) is moved + /// to the slowest-varying position first. + pub(crate) fn extensible_array( + cur_dims: &[u64], + max_dims: Option<&[u64]>, + chunk_dims: &[u64], + ) -> Result { + let unlim = max_dims.and_then(|m| m.iter().position(|&d| d == u64::MAX)); + Self::build(cur_dims, max_dims, chunk_dims, unlim) + } + + fn build( + cur_dims: &[u64], + max_dims: Option<&[u64]>, + chunk_dims: &[u64], + unlim: Option, + ) -> Result { + let rank = chunk_dims.len(); + if cur_dims.len() != rank || max_dims.is_some_and(|m| m.len() != rank) { + return Err(FormatError::ChunkedReadError( + "chunk index rank does not match the dataspace".into(), + )); + } + if chunk_dims.contains(&0) { + return Err(FormatError::ChunkedReadError( + "chunk dimension is zero".into(), + )); + } + let cur_chunks: Vec = cur_dims + .iter() + .zip(chunk_dims) + .map(|(&d, &c)| d.div_ceil(c)) + .collect(); + // Chunk counts of the maximum extent. An unlimited dimension has no + // finite count; it only ever sits in the slowest position, where its + // count never enters a stride. A (corrupt) maximum smaller than the + // current extent is widened so no allocated chunk becomes unreachable. + let max_chunks: Vec = (0..rank) + .map(|d| { + let max = max_dims.map_or(cur_dims[d], |m| m[d]); + if max == u64::MAX { + u64::MAX + } else { + max.div_ceil(chunk_dims[d]).max(cur_chunks[d]) + } + }) + .collect(); + + let mut order: Vec = (0..rank).collect(); + if let Some(u) = unlim { + order.remove(u); + order.insert(0, u); + } + let mut down = vec![1u64; rank]; + for p in (0..rank.saturating_sub(1)).rev() { + let next = max_chunks[order[p + 1]]; + if next == u64::MAX { + // Only reachable with more than one unlimited dimension, which + // neither index type can describe. + return Err(FormatError::ChunkedReadError( + "array chunk index with more than one unlimited dimension".into(), + )); + } + down[p] = down[p + 1].checked_mul(next).ok_or_else(|| { + FormatError::Overflow("chunk index linear stride overflows u64".into()) + })?; + } + Ok(Self { + chunk_dims: chunk_dims.to_vec(), + cur_chunks, + order, + down, + }) + } + + /// Dataset-space offsets of the chunk stored at linear `index`, or `None` + /// when that chunk lies outside the current extent (the index still has a + /// slot for it; the library ignores such chunks on read). + pub(crate) fn offsets(&self, index: u64) -> Option> { + let rank = self.chunk_dims.len(); + let mut offsets = vec![0u64; rank]; + let mut rem = index; + for p in 0..rank { + let d = self.order[p]; + let scaled = rem / self.down[p]; + rem %= self.down[p]; + if scaled >= self.cur_chunks[d] { + return None; + } + offsets[d] = scaled * self.chunk_dims[d]; + } + Some(offsets) + } + + /// Linear index of the chunk with scaled coordinates `scaled` + /// (`offset / chunk_dim` per dimension, in dataset order). + #[allow(dead_code)] // used by the writer + pub(crate) fn linear_index(&self, scaled: &[u64]) -> u64 { + self.order + .iter() + .zip(&self.down) + .map(|(&d, &stride)| scaled[d] * stride) + .sum() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn fixed_array_uses_max_dims() { + // shape (4, 6), chunks (2, 3), maxshape (20, 10): 10 x 4 chunk grid. + let g = ChunkGrid::fixed_array(&[4, 6], Some(&[20, 10]), &[2, 3]).unwrap(); + assert_eq!(g.offsets(0), Some(vec![0, 0])); + assert_eq!(g.offsets(1), Some(vec![0, 3])); + assert_eq!(g.offsets(2), None); // column chunk 2 is beyond the extent + assert_eq!(g.offsets(4), Some(vec![2, 0])); + assert_eq!(g.offsets(5), Some(vec![2, 3])); + assert_eq!(g.offsets(8), None); // row chunk 2 is beyond the extent + assert_eq!(g.linear_index(&[1, 1]), 5); + } + + #[test] + fn extensible_array_swizzles_unlimited_dim() { + // maxshape (10, None): dim 1 is unlimited and becomes slowest. + let g = ChunkGrid::extensible_array(&[4, 6], Some(&[10, u64::MAX]), &[2, 3]).unwrap(); + // max chunks of dim 0 = 5, so index = c1 * 5 + c0. + assert_eq!(g.linear_index(&[1, 0]), 1); + assert_eq!(g.linear_index(&[0, 1]), 5); + assert_eq!(g.offsets(5), Some(vec![0, 3])); + assert_eq!(g.offsets(6), Some(vec![2, 3])); + assert_eq!(g.offsets(2), None); + } + + #[test] + fn extensible_array_unlimited_first_is_row_major() { + let g = ChunkGrid::extensible_array(&[4, 6], Some(&[u64::MAX, 30]), &[2, 3]).unwrap(); + // max chunks of dim 1 = 10. + assert_eq!(g.linear_index(&[1, 1]), 11); + assert_eq!(g.offsets(11), Some(vec![2, 3])); + } + + #[test] + fn rejects_two_unlimited_dims_after_the_first() { + assert!(ChunkGrid::fixed_array(&[4, 6], Some(&[u64::MAX, u64::MAX]), &[2, 3]).is_err()); + } +} diff --git a/crates/clawhdf5-format/src/chunked_read.rs b/crates/clawhdf5-format/src/chunked_read.rs index 8dade54..f07ae27 100644 --- a/crates/clawhdf5-format/src/chunked_read.rs +++ b/crates/clawhdf5-format/src/chunked_read.rs @@ -593,6 +593,7 @@ pub fn list_chunks( file_data, &header, &dataspace.dimensions, + dataspace.max_dimensions.as_deref(), spatial_chunk_dims, elem_size as u32, offset_size, @@ -608,6 +609,7 @@ pub fn list_chunks( file_data, &header, &dataspace.dimensions, + dataspace.max_dimensions.as_deref(), spatial_chunk_dims, elem_size as u32, offset_size, diff --git a/crates/clawhdf5-format/src/extensible_array.rs b/crates/clawhdf5-format/src/extensible_array.rs index 4416a76..ba1c822 100644 --- a/crates/clawhdf5-format/src/extensible_array.rs +++ b/crates/clawhdf5-format/src/extensible_array.rs @@ -9,6 +9,7 @@ extern crate alloc; #[cfg(not(feature = "std"))] use alloc::{format, vec, vec::Vec}; +use crate::chunk_grid::ChunkGrid; use crate::chunked_read::ChunkInfo; use crate::error::FormatError; @@ -203,8 +204,7 @@ fn read_element( offset_size: u8, chunk_byte_size: u64, linear_index: usize, - num_chunks_per_dim: &[u64], - chunk_dimensions: &[u32], + grid: &ChunkGrid, ) -> Result<(Option, usize), FormatError> { let os = offset_size as usize; @@ -220,7 +220,10 @@ fn read_element( return Ok((None, os)); } let address = read_offset(data, pos, offset_size)?; - let offsets = index_to_chunk_offsets(linear_index, num_chunks_per_dim, chunk_dimensions); + // A slot beyond the current extent is ignored, as the library does. + let Some(offsets) = grid.offsets(linear_index as u64) else { + return Ok((None, os)); + }; Ok(( Some(ChunkInfo { chunk_size: chunk_byte_size as u32, @@ -261,7 +264,9 @@ fn read_element( data[fm_off + 2], data[fm_off + 3], ]); - let offsets = index_to_chunk_offsets(linear_index, num_chunks_per_dim, chunk_dimensions); + let Some(offsets) = grid.offsets(linear_index as u64) else { + return Ok((None, elem_total)); + }; Ok(( Some(ChunkInfo { chunk_size: chunk_size as u32, @@ -274,27 +279,6 @@ fn read_element( } } -/// Convert a linear chunk index to N-dimensional chunk offsets in dataset space. -fn index_to_chunk_offsets( - index: usize, - num_chunks_per_dim: &[u64], - chunk_dimensions: &[u32], -) -> Vec { - let rank = num_chunks_per_dim.len(); - let mut offsets = vec![0u64; rank]; - let mut remaining = index as u64; - for d in (0..rank).rev() { - let nchunks = num_chunks_per_dim[d]; - if nchunks == 0 { - continue; - } - let chunk_idx = remaining % nchunks; - remaining /= nchunks; - offsets[d] = chunk_idx * chunk_dimensions[d] as u64; - } - offsets -} - /// Collect elements from a data block at the given offset. #[allow(clippy::too_many_arguments)] /// Layout of super block `u`, per the HDF5 spec: the number of data blocks it @@ -339,8 +323,7 @@ fn read_data_block_elements( offset_size: u8, chunk_byte_size: u64, start_index: usize, - num_chunks_per_dim: &[u64], - chunk_dimensions: &[u32], + grid: &ChunkGrid, page_init: &[u8], first_page: usize, ) -> Result, FormatError> { @@ -376,8 +359,7 @@ fn read_data_block_elements( offset_size, chunk_byte_size, first_index + i, - num_chunks_per_dim, - chunk_dimensions, + grid, )?; if let Some(ci) = info { chunks.push(ci); @@ -449,25 +431,19 @@ pub fn read_extensible_array_chunks( file_data: &[u8], header: &ExtensibleArrayHeader, dataset_dims: &[u64], + max_dims: Option<&[u64]>, chunk_dimensions: &[u32], element_size: u32, offset_size: u8, _length_size: u8, ) -> Result, FormatError> { - let rank = chunk_dimensions.len(); let os = offset_size as usize; - let mut num_chunks_per_dim = Vec::with_capacity(rank); - for d in 0..rank { - let ch_dim = chunk_dimensions[d] as u64; - if ch_dim == 0 { - return Err(FormatError::ChunkedReadError( - "chunk dimension is zero".into(), - )); - } - let ds_dim = dataset_dims[d]; - num_chunks_per_dim.push(ds_dim.div_ceil(ch_dim)); - } + // Linear indexes follow the maximum dimensions, with the unlimited + // dimension swizzled to the slowest position (see `chunk_grid`). + let dims_u64: Vec = chunk_dimensions.iter().map(|&d| d as u64).collect(); + let grid = ChunkGrid::extensible_array(dataset_dims, max_dims, &dims_u64)?; + let grid = &grid; let chunk_byte_size: u64 = chunk_dimensions.iter().map(|&d| d as u64).product::() * element_size as u64; @@ -557,8 +533,7 @@ pub fn read_extensible_array_chunks( offset_size, chunk_byte_size, i, - &num_chunks_per_dim, - chunk_dimensions, + grid, )?; if let Some(ci) = info { chunks.push(ci); @@ -594,8 +569,7 @@ pub fn read_extensible_array_chunks( offset_size, chunk_byte_size, global_index, - &num_chunks_per_dim, - chunk_dimensions, + grid, &[], 0, )?); @@ -625,8 +599,7 @@ pub fn read_extensible_array_chunks( offset_size, chunk_byte_size, global_index, - &num_chunks_per_dim, - chunk_dimensions, + grid, )?); } global_index = @@ -653,8 +626,7 @@ fn read_super_block( offset_size: u8, chunk_byte_size: u64, start_index: usize, - num_chunks_per_dim: &[u64], - chunk_dimensions: &[u32], + grid: &ChunkGrid, ) -> Result, FormatError> { let os = offset_size as usize; let sb_header_size = 4 + 1 + 1 + os + arr_off_size(header); @@ -710,8 +682,7 @@ fn read_super_block( offset_size, chunk_byte_size, global_idx, - num_chunks_per_dim, - chunk_dimensions, + grid, bitmap, i * npages, )?); @@ -735,35 +706,18 @@ mod tests { } #[test] fn index_to_offsets_1d() { - let num_chunks = vec![5u64]; - let chunk_dims = vec![20u32]; - assert_eq!(index_to_chunk_offsets(0, &num_chunks, &chunk_dims), vec![0]); - assert_eq!( - index_to_chunk_offsets(1, &num_chunks, &chunk_dims), - vec![20] - ); - assert_eq!( - index_to_chunk_offsets(4, &num_chunks, &chunk_dims), - vec![80] - ); + let g = ChunkGrid::fixed_array(&[100], None, &[20]).unwrap(); + assert_eq!(g.offsets(0).unwrap(), vec![0]); + assert_eq!(g.offsets(1).unwrap(), vec![20]); + assert_eq!(g.offsets(4).unwrap(), vec![80]); } #[test] fn index_to_offsets_2d() { - let num_chunks = vec![3u64, 2]; - let chunk_dims = vec![4u32, 3]; - assert_eq!( - index_to_chunk_offsets(0, &num_chunks, &chunk_dims), - vec![0, 0] - ); - assert_eq!( - index_to_chunk_offsets(1, &num_chunks, &chunk_dims), - vec![0, 3] - ); - assert_eq!( - index_to_chunk_offsets(2, &num_chunks, &chunk_dims), - vec![4, 0] - ); + let g = ChunkGrid::fixed_array(&[10, 6], None, &[4, 3]).unwrap(); + assert_eq!(g.offsets(0).unwrap(), vec![0, 0]); + assert_eq!(g.offsets(1).unwrap(), vec![0, 3]); + assert_eq!(g.offsets(2).unwrap(), vec![4, 0]); } #[test] @@ -830,7 +784,7 @@ mod tests { index_block_address: (usize::MAX - 4) as u64, }; let buf = vec![0u8; 64]; - let r = read_extensible_array_chunks(&buf, &header, &[100], &[20], 8, 8, 8); + let r = read_extensible_array_chunks(&buf, &header, &[100], None, &[20], 8, 8, 8); assert!(r.is_err()); } @@ -913,9 +867,17 @@ mod tests { let header = ExtensibleArrayHeader::parse(&file_data, aehd_offset, os, ls).unwrap(); let ds_dims = vec![40u64]; // 2 chunks × 20 elements let chunk_dims = vec![20u32]; - let chunks = - read_extensible_array_chunks(&file_data, &header, &ds_dims, &chunk_dims, 8, os, ls) - .unwrap(); + let chunks = read_extensible_array_chunks( + &file_data, + &header, + &ds_dims, + None, + &chunk_dims, + 8, + os, + ls, + ) + .unwrap(); assert_eq!(chunks.len(), 2); assert_eq!(chunks[0].address, base_addr); @@ -1023,9 +985,17 @@ mod tests { let header = ExtensibleArrayHeader::parse(&file_data, aehd_offset, os, ls).unwrap(); let ds_dims = vec![40u64]; let chunk_dims = vec![10u32]; - let chunks = - read_extensible_array_chunks(&file_data, &header, &ds_dims, &chunk_dims, 8, os, ls) - .unwrap(); + let chunks = read_extensible_array_chunks( + &file_data, + &header, + &ds_dims, + None, + &chunk_dims, + 8, + os, + ls, + ) + .unwrap(); assert_eq!(chunks.len(), 4); for (i, c) in chunks.iter().enumerate() { @@ -1047,10 +1017,8 @@ mod tests { #[test] fn read_element_unallocated() { let data = vec![0xFFu8; 16]; - let num_chunks = vec![5u64]; - let chunk_dims = vec![10u32]; - let (info, consumed) = - read_element(&data, 0, 0, 8, 8, 80, 0, &num_chunks, &chunk_dims).unwrap(); + let grid = ChunkGrid::fixed_array(&[50], None, &[10]).unwrap(); + let (info, consumed) = read_element(&data, 0, 0, 8, 8, 80, 0, &grid).unwrap(); assert!(info.is_none()); assert_eq!(consumed, 8); } @@ -1069,20 +1037,9 @@ mod tests { // Filter mask data[12..16].copy_from_slice(&0u32.to_le_bytes()); - let num_chunks = vec![5u64]; - let chunk_dims = vec![10u32]; - let (info, consumed) = read_element( - &data, - 0, - 1, - elem_size as u8, - os, - 80, - 2, - &num_chunks, - &chunk_dims, - ) - .unwrap(); + let grid = ChunkGrid::fixed_array(&[50], None, &[10]).unwrap(); + let (info, consumed) = + read_element(&data, 0, 1, elem_size as u8, os, 80, 2, &grid).unwrap(); let ci = info.unwrap(); assert_eq!(ci.address, 0x2000); assert_eq!(ci.chunk_size, 120); diff --git a/crates/clawhdf5-format/src/fixed_array.rs b/crates/clawhdf5-format/src/fixed_array.rs index 4c79f03..8546224 100644 --- a/crates/clawhdf5-format/src/fixed_array.rs +++ b/crates/clawhdf5-format/src/fixed_array.rs @@ -6,6 +6,7 @@ extern crate alloc; #[cfg(not(feature = "std"))] use alloc::{format, vec, vec::Vec}; +use crate::chunk_grid::ChunkGrid; use crate::chunked_read::ChunkInfo; use crate::error::FormatError; @@ -151,13 +152,13 @@ pub fn read_fixed_array_chunks( file_data: &[u8], header: &FixedArrayHeader, dataset_dims: &[u64], + max_dims: Option<&[u64]>, chunk_dimensions: &[u32], element_size: u32, offset_size: u8, _length_size: u8, ) -> Result, FormatError> { let db_offset = header.data_block_address as usize; - let rank = chunk_dimensions.len(); // Parse data block header: FADB(4) + version(1) + client_id(1) + header_address(offset_size) let db_header_size = 4 + 1 + 1 + offset_size as usize; @@ -198,19 +199,10 @@ pub fn read_fixed_array_chunks( )) }; - // Compute chunk offsets based on index. - // Chunks are stored in row-major order within the dataset space. - let mut num_chunks_per_dim = Vec::with_capacity(rank); - for d_idx in 0..rank { - let ch_dim = chunk_dimensions[d_idx] as u64; - if ch_dim == 0 { - return Err(FormatError::ChunkedReadError( - "chunk dimension is zero".into(), - )); - } - let ds_dim = dataset_dims[d_idx]; - num_chunks_per_dim.push(ds_dim.div_ceil(ch_dim)); - } + // The index is laid out over the chunk grid of the *maximum* dimensions + // (row-major), so a dataset smaller than its maxshape has gaps. + let dims_u64: Vec = chunk_dimensions.iter().map(|&d| d as u64).collect(); + let grid = ChunkGrid::fixed_array(dataset_dims, max_dims, &dims_u64)?; let chunk_byte_size: u64 = chunk_dimensions.iter().map(|&d| d as u64).product::() * element_size as u64; @@ -226,7 +218,11 @@ pub fn read_fixed_array_chunks( header.element_size, chunk_byte_size, )? { - let offsets = index_to_chunk_offsets(i, &num_chunks_per_dim, chunk_dimensions); + // A slot beyond the current extent is ignored, as the + // library does. + let Some(offsets) = grid.offsets(i as u64) else { + return Ok(()); + }; chunks.push(ChunkInfo { chunk_size, filter_mask, @@ -367,27 +363,6 @@ fn parse_fa_element( } } -/// Convert a linear chunk index to N-dimensional chunk offsets in dataset space. -fn index_to_chunk_offsets( - index: usize, - num_chunks_per_dim: &[u64], - chunk_dimensions: &[u32], -) -> Vec { - let rank = num_chunks_per_dim.len(); - let mut offsets = vec![0u64; rank]; - let mut remaining = index as u64; - for d in (0..rank).rev() { - let nchunks = num_chunks_per_dim[d]; - if nchunks == 0 { - continue; - } - let chunk_idx = remaining % nchunks; - remaining /= nchunks; - offsets[d] = chunk_idx * chunk_dimensions[d] as u64; - } - offsets -} - /// Read a variable-length little-endian unsigned integer. fn read_variable_length(data: &[u8], size: usize) -> Result { if size > 8 || data.len() < size { @@ -416,44 +391,21 @@ mod tests { #[test] fn index_to_offsets_1d() { - let num_chunks = vec![5u64]; - let chunk_dims = vec![20u32]; - assert_eq!(index_to_chunk_offsets(0, &num_chunks, &chunk_dims), vec![0]); - assert_eq!( - index_to_chunk_offsets(1, &num_chunks, &chunk_dims), - vec![20] - ); - assert_eq!( - index_to_chunk_offsets(4, &num_chunks, &chunk_dims), - vec![80] - ); + let g = ChunkGrid::fixed_array(&[100], None, &[20]).unwrap(); + assert_eq!(g.offsets(0).unwrap(), vec![0]); + assert_eq!(g.offsets(1).unwrap(), vec![20]); + assert_eq!(g.offsets(4).unwrap(), vec![80]); } #[test] fn index_to_offsets_2d() { // 10x6 dataset with 4x3 chunks => ceil(10/4)=3, ceil(6/3)=2 => 6 chunks - let num_chunks = vec![3u64, 2]; - let chunk_dims = vec![4u32, 3]; - assert_eq!( - index_to_chunk_offsets(0, &num_chunks, &chunk_dims), - vec![0, 0] - ); - assert_eq!( - index_to_chunk_offsets(1, &num_chunks, &chunk_dims), - vec![0, 3] - ); - assert_eq!( - index_to_chunk_offsets(2, &num_chunks, &chunk_dims), - vec![4, 0] - ); - assert_eq!( - index_to_chunk_offsets(3, &num_chunks, &chunk_dims), - vec![4, 3] - ); - assert_eq!( - index_to_chunk_offsets(5, &num_chunks, &chunk_dims), - vec![8, 3] - ); + let g = ChunkGrid::fixed_array(&[10, 6], None, &[4, 3]).unwrap(); + assert_eq!(g.offsets(0).unwrap(), vec![0, 0]); + assert_eq!(g.offsets(1).unwrap(), vec![0, 3]); + assert_eq!(g.offsets(2).unwrap(), vec![4, 0]); + assert_eq!(g.offsets(3).unwrap(), vec![4, 3]); + assert_eq!(g.offsets(5).unwrap(), vec![8, 3]); } #[test] @@ -517,7 +469,7 @@ mod tests { let read = |f: &[u8], fahd: usize| -> Result, FormatError> { let h = FixedArrayHeader::parse(f, fahd, 8, 8)?; - read_fixed_array_chunks(f, &h, &[60], &[20], 8, 8, 8) + read_fixed_array_chunks(f, &h, &[60], None, &[20], 8, 8, 8) }; let (clean, fahd) = build(); @@ -562,7 +514,7 @@ mod tests { let db = 0x100usize; buf[db..db + 4].copy_from_slice(b"FADB"); let header = FixedArrayHeader::parse(&buf, fahd, 8, 8).unwrap(); - let r = read_fixed_array_chunks(&buf, &header, &[100], &[20], 8, 8, 8); + let r = read_fixed_array_chunks(&buf, &header, &[100], None, &[20], 8, 8, 8); assert!(r.is_err()); } @@ -579,7 +531,7 @@ mod tests { stamp_checksum(&mut buf, fahd, fahd + 24); buf[0x80..0x84].copy_from_slice(b"FADB"); let header = FixedArrayHeader::parse(&buf, fahd, 8, 8).unwrap(); - let r = read_fixed_array_chunks(&buf, &header, &[100], &[20], 8, 8, 8); + let r = read_fixed_array_chunks(&buf, &header, &[100], None, &[20], 8, 8, 8); assert!(r.is_err()); } @@ -602,7 +554,7 @@ mod tests { data_block_address: (usize::MAX - 4) as u64, }; let buf = vec![0u8; 64]; - let r = read_fixed_array_chunks(&buf, &header, &[100], &[20], 8, 8, 8); + let r = read_fixed_array_chunks(&buf, &header, &[100], None, &[20], 8, 8, 8); assert!(r.is_err()); } @@ -664,6 +616,7 @@ mod tests { &file_data, &header, &ds_dims, + None, &chunk_dims, 8, offset_size, @@ -740,6 +693,7 @@ mod tests { &file_data, &header, &ds_dims, + None, &chunk_dims, 8, offset_size, @@ -840,6 +794,7 @@ mod tests { &file_data, &header, &ds_dims, + None, &chunk_dims, 8, offset_size, diff --git a/crates/clawhdf5-format/src/lib.rs b/crates/clawhdf5-format/src/lib.rs index 54bd326..4a6c2a1 100644 --- a/crates/clawhdf5-format/src/lib.rs +++ b/crates/clawhdf5-format/src/lib.rs @@ -54,6 +54,7 @@ pub mod btree_v1; pub mod btree_v2; pub mod checksum; pub mod chunk_cache; +mod chunk_grid; pub mod chunk_index; pub mod chunked_read; pub mod chunked_write; diff --git a/crates/clawhdf5/tests/chunk_index_interop.rs b/crates/clawhdf5/tests/chunk_index_interop.rs new file mode 100644 index 0000000..9c030a3 --- /dev/null +++ b/crates/clawhdf5/tests/chunk_index_interop.rs @@ -0,0 +1,256 @@ +//! Fixed Array / Extensible Array chunk-index interop with libhdf5 (via h5py). +//! +//! Both indexes place each chunk at a linear index computed from the +//! dataset's *maximum* dimensions, and the Extensible Array additionally +//! moves its unlimited dimension to the slowest-varying position. Getting +//! either wrong reads (or writes) every chunk after the first row in the +//! wrong place, silently, so these tests compare every value. +//! +//! Skipped when python3 with h5py is unavailable, unless +//! `CLAWHDF5_REQUIRE_INTEROP=1`. + +use std::process::Command; + +use clawhdf5::File; + +fn python() -> String { + std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) +} + +fn interop_required() -> bool { + std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1") +} + +fn python_available() -> bool { + Command::new(python()) + .args(["-c", "import h5py"]) + .output() + .map(|o| o.status.success()) + .unwrap_or(false) +} + +macro_rules! skip_if_no_python { + () => { + if !python_available() { + assert!( + !interop_required(), + "CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available" + ); + eprintln!("SKIP: python3 with h5py not available"); + return; + } + }; +} + +fn run_python(script: &str) -> String { + let output = Command::new(python()) + .args(["-c", script]) + .output() + .expect("failed to run python"); + if !output.status.success() { + panic!( + "Python script failed:\nSTDOUT: {}\nSTDERR: {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } + String::from_utf8_lossy(&output.stdout).trim().to_string() +} + +/// Row-major `arange` of `shape`, cropped to `crop` (the current extent). +fn arange_cropped(full: &[usize], crop: &[usize]) -> Vec { + let n: usize = crop.iter().product(); + let mut out = Vec::with_capacity(n); + for flat in 0..n { + let mut rem = flat; + let mut src = 0usize; + let mut stride = 1usize; + let mut coords = vec![0usize; crop.len()]; + for d in (0..crop.len()).rev() { + coords[d] = rem % crop[d]; + rem /= crop[d]; + } + for d in (0..full.len()).rev() { + src += coords[d] * stride; + stride *= full[d]; + } + out.push(src as i32); + } + out +} + +/// One `i4` dataset, filled with `arange` over `full` and then resized to +/// `shape` (equal to `full` unless the case shrinks it). +struct Case { + name: &'static str, + full: Vec, + shape: Vec, + chunks: Vec, + maxshape: &'static str, + extra: &'static str, + index: &'static str, +} + +fn py_tuple(v: &[usize]) -> String { + let parts: Vec = v.iter().map(|x| x.to_string()).collect(); + format!("({},)", parts.join(",")) +} + +/// Have h5py (`libver="latest"`, so Fixed/Extensible Array indexes) write +/// every case to one file, then read each back and compare every value. +fn check_h5py_written(cases: &[Case]) { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("h5py_chunk_index.h5"); + let path_str = path.display().to_string(); + + let mut script = + format!("import h5py, numpy as np\nf = h5py.File(r'{path_str}', 'w', libver='latest')\n"); + for c in cases { + script += &format!( + "d = f.create_dataset('{name}', data=np.arange({n}, dtype='i4').reshape({full}), \ + chunks={chunks}, maxshape={maxshape}{extra})\n\ + d.resize({shape})\n", + name = c.name, + n = c.full.iter().product::(), + full = py_tuple(&c.full), + chunks = py_tuple(&c.chunks), + maxshape = c.maxshape, + extra = c.extra, + shape = py_tuple(&c.shape), + ); + } + script += "f.close()\n"; + run_python(&script); + + let file = File::open(&path).unwrap(); + for c in cases { + let ds = file.dataset(c.name).unwrap(); + let shape: Vec = ds.shape().unwrap().iter().map(|&d| d as usize).collect(); + assert_eq!(shape, c.shape, "{}: shape", c.name); + let got = ds.read_i32().unwrap(); + let want = arange_cropped(&c.full, &c.shape); + let bad = got.iter().zip(&want).filter(|(a, b)| a != b).count(); + assert_eq!( + got, + want, + "{}: {bad} of {} values differ (index {})", + c.name, + want.len(), + c.index + ); + } +} + +/// h5py-written Extensible Array whose unlimited dimension is not the first, +/// with the current shape smaller than the finite maximum: the library +/// swizzles the unlimited dimension to the slowest position and strides the +/// rest by their maximum chunk counts. +#[test] +fn h5py_extensible_array_partial_extent_reads_correctly() { + skip_if_no_python!(); + check_h5py_written(&[ + // The `ea_fa_partial.h5` repro from the conformance sweep. + Case { + name: "ea_10_none", + full: vec![4, 6], + shape: vec![4, 6], + chunks: vec![2, 3], + maxshape: "(10, None)", + extra: "", + index: "EA, unlimited dim 1", + }, + Case { + name: "ea_none_10", + full: vec![4, 6], + shape: vec![4, 6], + chunks: vec![2, 3], + maxshape: "(None, 10)", + extra: "", + index: "EA, unlimited dim 0", + }, + Case { + name: "ea_3d_mid", + full: vec![3, 4, 5], + shape: vec![3, 4, 5], + chunks: vec![2, 3, 2], + maxshape: "(5, None, 7)", + extra: "", + index: "EA, unlimited dim 1 of 3", + }, + Case { + name: "ea_3d_last_gzip", + full: vec![3, 4, 5], + shape: vec![3, 4, 5], + chunks: vec![2, 3, 2], + maxshape: "(5, 9, None)", + extra: ", compression='gzip'", + index: "EA, unlimited dim 2 of 3, filtered", + }, + // Many chunks: crosses data blocks, super blocks and paging. + Case { + name: "ea_many", + full: vec![3, 1500], + shape: vec![3, 1500], + chunks: vec![1, 1], + maxshape: "(4, None)", + extra: "", + index: "EA, 4500 slots", + }, + // Shrunk after writing: chunks beyond the extent must be ignored. + Case { + name: "ea_shrunk", + full: vec![8, 9], + shape: vec![3, 4], + chunks: vec![2, 3], + maxshape: "(10, None)", + extra: "", + index: "EA, shrunk", + }, + ]); +} + +/// h5py-written Fixed Array with the current shape smaller than a finite +/// maxshape: the index has one slot per chunk of the *maximum* extent. +#[test] +fn h5py_fixed_array_partial_extent_reads_correctly() { + skip_if_no_python!(); + check_h5py_written(&[ + Case { + name: "fa_20_10", + full: vec![4, 6], + shape: vec![4, 6], + chunks: vec![2, 3], + maxshape: "(20, 10)", + extra: "", + index: "FA", + }, + Case { + name: "fa_3d_gzip", + full: vec![3, 4, 5], + shape: vec![3, 4, 5], + chunks: vec![2, 3, 2], + maxshape: "(6, 8, 10)", + extra: ", compression='gzip'", + index: "FA, filtered", + }, + // Paged (> 1024 slots) with most of them beyond the extent. + Case { + name: "fa_paged", + full: vec![30, 50], + shape: vec![30, 50], + chunks: vec![1, 1], + maxshape: "(40, 60)", + extra: "", + index: "FA, 2400 slots, paged", + }, + Case { + name: "fa_shrunk", + full: vec![8, 9], + shape: vec![5, 2], + chunks: vec![2, 3], + maxshape: "(20, 10)", + extra: "", + index: "FA, shrunk", + }, + ]); +}