diff --git a/crates/clawhdf5-format/src/extensible_array.rs b/crates/clawhdf5-format/src/extensible_array.rs index ba1c822..529955d 100644 --- a/crates/clawhdf5-format/src/extensible_array.rs +++ b/crates/clawhdf5-format/src/extensible_array.rs @@ -12,16 +12,19 @@ use alloc::{format, vec, vec::Vec}; use crate::chunk_grid::ChunkGrid; use crate::chunked_read::ChunkInfo; use crate::error::FormatError; +use crate::storage::{Storage, Window, read_exact_at}; /// Verify the Jenkins lookup3 checksum stored immediately after -/// `data[start..end]`, as every Extensible Array structure carries one. +/// `data[start..end]`, as every Extensible Array structure carries one. `w` +/// is a window of the file and `start`/`end` are relative to it. /// /// A corrupt chunk index yields addresses pointing at the wrong bytes, so a /// mismatch is an error: otherwise the damage surfaces as plausible data read /// from the wrong chunk. #[cfg(feature = "checksum")] -fn verify_checksum(data: &[u8], start: usize, end: usize) -> Result<(), FormatError> { - ensure_len(data, end, 4)?; +fn verify_checksum(w: &Window<'_>, start: usize, end: usize) -> Result<(), FormatError> { + w.ensure(end, 4)?; + let data: &[u8] = &w.bytes; let stored = u32::from_le_bytes([data[end], data[end + 1], data[end + 2], data[end + 3]]); let computed = crate::checksum::jenkins_lookup3(&data[start..end]); if computed != stored { @@ -34,7 +37,7 @@ fn verify_checksum(data: &[u8], start: usize, end: usize) -> Result<(), FormatEr } #[cfg(not(feature = "checksum"))] -fn verify_checksum(_data: &[u8], _start: usize, _end: usize) -> Result<(), FormatError> { +fn verify_checksum(_w: &Window<'_>, _start: usize, _end: usize) -> Result<(), FormatError> { Ok(()) } @@ -80,19 +83,6 @@ fn read_offset(data: &[u8], pos: usize, size: u8) -> Result { }) } -fn ensure_len(data: &[u8], offset: usize, needed: usize) -> Result<(), FormatError> { - if offset - .checked_add(needed) - .is_none_or(|end| end > data.len()) - { - return Err(FormatError::UnexpectedEof { - expected: offset.saturating_add(needed), - available: data.len(), - }); - } - Ok(()) -} - fn is_undefined_addr(addr: u64, offset_size: u8) -> bool { match offset_size { 2 => addr == 0xFFFF, @@ -130,6 +120,16 @@ impl ExtensibleArrayHeader { offset: usize, offset_size: u8, length_size: u8, + ) -> Result { + Self::parse_in(&file_data, offset as u64, offset_size, length_size) + } + + /// [`Self::parse`] over any [`Storage`]: one read of the header. + pub fn parse_in( + file: &dyn Storage, + offset: u64, + offset_size: u8, + length_size: u8, ) -> Result { // EAHD: signature(4) + version(1) + client_id(1) + element_size(1) + // max_nelmts_bits(1) + idx_blk_elmts(1) + min_dblk_nelmts(1) + @@ -137,9 +137,10 @@ impl ExtensibleArrayHeader { // 6 stats fields (each length_size) + index_block_address(offset_size) + checksum(4) let min_size = 4 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 6 * length_size as usize + offset_size as usize + 4; - ensure_len(file_data, offset, min_size)?; + let w = Window::read(file, offset, min_size)?; + w.ensure(0, min_size)?; - let d = &file_data[offset..]; + let d: &[u8] = &w.bytes; if &d[0..4] != b"EAHD" { return Err(FormatError::ChunkedReadError( "invalid Extensible Array header signature".into(), @@ -172,7 +173,7 @@ impl ExtensibleArrayHeader { pos += ls; // skip max_idx_set (6th stats field) let index_block_address = read_offset(d, pos, offset_size)?; pos += offset_size as usize; - verify_checksum(file_data, offset, offset + pos)?; + verify_checksum(&w, 0, pos)?; Ok(ExtensibleArrayHeader { client_id, @@ -193,11 +194,11 @@ impl ExtensibleArrayHeader { } } -/// Read a single element from the extensible array element data. +/// Read a single element at offset `pos` of the window `w`. /// Returns (chunk_info, bytes_consumed) or None if unallocated. #[allow(clippy::too_many_arguments)] fn read_element( - data: &[u8], + w: &Window<'_>, pos: usize, client_id: u8, element_size: u8, @@ -207,15 +208,11 @@ fn read_element( grid: &ChunkGrid, ) -> Result<(Option, usize), FormatError> { let os = offset_size as usize; + let data: &[u8] = &w.bytes; if client_id == 0 { // Non-filtered: just address - if pos + os > data.len() { - return Err(FormatError::UnexpectedEof { - expected: pos + os, - available: data.len(), - }); - } + w.ensure(pos, os)?; if is_undefined(data, pos, offset_size) { return Ok((None, os)); } @@ -243,15 +240,7 @@ fn read_element( } let chunk_size_bytes = es - os - 4; let elem_total = os + chunk_size_bytes + 4; - if pos - .checked_add(elem_total) - .is_none_or(|end| end > data.len()) - { - return Err(FormatError::UnexpectedEof { - expected: pos.saturating_add(elem_total), - available: data.len(), - }); - } + w.ensure(pos, elem_total)?; if is_undefined(data, pos, offset_size) { return Ok((None, elem_total)); } @@ -316,8 +305,8 @@ fn page_nelmts(header: &ExtensibleArrayHeader) -> Option { /// stores only its prefix, then one slot per page. #[allow(clippy::too_many_arguments)] fn read_data_block_elements( - file_data: &[u8], - db_offset: usize, + file: &dyn Storage, + db_offset: u64, nelmts: usize, header: &ExtensibleArrayHeader, offset_size: u8, @@ -330,21 +319,28 @@ fn read_data_block_elements( // EADB: signature(4) + version(1) + client_id(1) + header_address(offset_size) // + block offset(arr_off_size) let db_header_size = 4 + 1 + 1 + offset_size as usize + arr_off_size(header); - ensure_len(file_data, db_offset, db_header_size)?; + let prefix = read_exact_at(file, db_offset, db_header_size)?; - if &file_data[db_offset..db_offset + 4] != b"EADB" { + if &prefix[0..4] != b"EADB" { return Err(FormatError::ChunkedReadError( "invalid Extensible Array data block signature".into(), )); } - let mut pos = db_offset + db_header_size; + // Positions below are relative to the data block. + let mut pos = db_header_size; let page = page_nelmts(header).ok_or_else(|| { FormatError::Overflow("Extensible Array page element count overflows usize".into()) })?; + let elem_bytes = if header.client_id == 0 { + offset_size as usize + } else { + header.element_size as usize + }; let mut chunks = Vec::new(); - let read_run = |from: usize, + let read_run = |w: &Window<'_>, + from: usize, count: usize, first_index: usize, chunks: &mut Vec| @@ -352,7 +348,7 @@ fn read_data_block_elements( let mut p = from; for i in 0..count { let (info, consumed) = read_element( - file_data, + w, p, header.client_id, header.element_size, @@ -370,18 +366,16 @@ fn read_data_block_elements( }; if nelmts <= page { - // Prefix and elements are covered by one checksum. - let elem_bytes = if header.client_id == 0 { - offset_size as usize - } else { - header.element_size as usize - }; + // Prefix and elements are covered by one checksum. One window holds + // all of it (or ends at the end of the file), so its bounds checks + // are the whole-file ones. let end = nelmts .checked_mul(elem_bytes) .and_then(|b| pos.checked_add(b)) .ok_or_else(|| FormatError::Overflow("Extensible Array data block span".into()))?; - verify_checksum(file_data, db_offset, end)?; - read_run(pos, nelmts, start_index, &mut chunks)?; + let w = Window::read(file, db_offset, end.saturating_add(4))?; + verify_checksum(&w, 0, end)?; + read_run(&w, pos, nelmts, start_index, &mut chunks)?; return Ok(chunks); } @@ -389,18 +383,19 @@ fn read_data_block_elements( // each holding `page` elements followed by a checksum. Pages whose bit is // clear were never written; their slot still occupies the file, so stride // over it rather than reading zeros as addresses. - verify_checksum(file_data, db_offset, pos)?; + let npages = nelmts.div_ceil(page); + // The whole data block in one window: every position checked below lies + // inside it (or past the end of the file). + let block_len = pos + .saturating_add(4) + .saturating_add(npages.saturating_mul(page.saturating_mul(elem_bytes).saturating_add(4))); + let w = Window::read(file, db_offset, block_len)?; + verify_checksum(&w, 0, pos)?; pos += 4; - let elem_bytes = if header.client_id == 0 { - offset_size as usize - } else { - header.element_size as usize - }; let page_stride = page .checked_mul(elem_bytes) .and_then(|b| b.checked_add(4)) .ok_or_else(|| FormatError::Overflow("Extensible Array page stride".into()))?; - let npages = nelmts.div_ceil(page); for p in 0..npages { // One bit per page across the whole super block, packed contiguously // and MSB-first within each byte, as H5VM_bit_get reads it. @@ -412,8 +407,8 @@ fn read_data_block_elements( let count = core::cmp::min(page, nelmts - p * page); // Each page carries its own checksum, over a full page's worth of // slots even when the last one holds fewer live elements. - verify_checksum(file_data, pos, pos + page * elem_bytes)?; - read_run(pos, count, start_index + p * page, &mut chunks)?; + verify_checksum(&w, pos, pos + page * elem_bytes)?; + read_run(&w, pos, count, start_index + p * page, &mut chunks)?; } pos = pos .checked_add(page_stride) @@ -435,6 +430,32 @@ pub fn read_extensible_array_chunks( chunk_dimensions: &[u32], element_size: u32, offset_size: u8, + length_size: u8, +) -> Result, FormatError> { + read_extensible_array_chunks_in( + &file_data, + header, + dataset_dims, + max_dims, + chunk_dimensions, + element_size, + offset_size, + length_size, + ) +} + +/// [`read_extensible_array_chunks`] over any [`Storage`]: one read of the +/// index block's prefix, one of the whole index block, and the same for +/// every super block and data block it references. +#[allow(clippy::too_many_arguments)] +pub fn read_extensible_array_chunks_in( + file: &dyn Storage, + header: &ExtensibleArrayHeader, + dataset_dims: &[u64], + max_dims: Option<&[u64]>, + chunk_dimensions: &[u32], + element_size: u32, + offset_size: u8, _length_size: u8, ) -> Result, FormatError> { let os = offset_size as usize; @@ -451,16 +472,17 @@ pub fn read_extensible_array_chunks( // Parse index block (EAIB): signature(4) + version(1) + client_id(1) // + header address(offset_size), then the inline elements, then the // direct data block addresses, then the super block addresses. - let ib_offset = header.index_block_address as usize; + // Positions below are relative to the index block. + let ib_offset = header.index_block_address; let ib_header_size = 4 + 1 + 1 + os; - ensure_len(file_data, ib_offset, ib_header_size)?; + let prefix = read_exact_at(file, ib_offset, ib_header_size)?; - if &file_data[ib_offset..ib_offset + 4] != b"EAIB" { + if &prefix[0..4] != b"EAIB" { return Err(FormatError::ChunkedReadError( "invalid Extensible Array index block signature".into(), )); } - let mut pos = ib_offset + ib_header_size; + let mut pos = ib_header_size; let mut chunks = Vec::new(); let total_elements = header.num_elements as usize; @@ -520,13 +542,16 @@ pub fn read_extensible_array_chunks( .and_then(|n| n.checked_mul(os).and_then(|b| p.checked_add(b))) }) .ok_or_else(|| FormatError::Overflow("Extensible Array index block span".into()))?; - verify_checksum(file_data, ib_offset, ib_end)?; + // The whole index block in one window: every position read below is + // before `ib_end`. + let w = Window::read(file, ib_offset, ib_end.saturating_add(4))?; + verify_checksum(&w, 0, ib_end)?; // 1. Elements stored inline in the index block. let n_inline = (header.idx_blk_elmts as usize).min(total_elements); for i in 0..n_inline { let (info, consumed) = read_element( - file_data, + &w, pos, header.client_id, header.element_size, @@ -550,8 +575,8 @@ pub fn read_extensible_array_chunks( if global_index >= total_elements { return Ok(chunks); } - ensure_len(file_data, pos, os)?; - let addr = read_offset(file_data, pos, offset_size)?; + w.ensure(pos, os)?; + let addr = read_offset(&w.bytes, pos, offset_size)?; pos += os; if !is_undefined_addr(addr, offset_size) { if dblk_nelmts > page_nelmts(header).unwrap_or(usize::MAX) { @@ -562,8 +587,8 @@ pub fn read_extensible_array_chunks( )); } chunks.extend(read_data_block_elements( - file_data, - addr as usize, + file, + addr, dblk_nelmts, header, offset_size, @@ -583,16 +608,16 @@ pub fn read_extensible_array_chunks( if global_index >= total_elements { break; } - ensure_len(file_data, pos, os)?; - let sb_addr = read_offset(file_data, pos, offset_size)?; + w.ensure(pos, os)?; + let sb_addr = read_offset(&w.bytes, pos, offset_size)?; pos += os; let (ndblks, dblk_nelmts) = sblk_info(u, dmin).ok_or_else(|| { FormatError::Overflow("Extensible Array super block layout overflows usize".into()) })?; if !is_undefined_addr(sb_addr, offset_size) { chunks.extend(read_super_block( - file_data, - sb_addr as usize, + file, + sb_addr, ndblks, dblk_nelmts, header, @@ -618,8 +643,8 @@ pub fn read_extensible_array_chunks( /// + one address per data block + checksum. #[allow(clippy::too_many_arguments)] fn read_super_block( - file_data: &[u8], - sb_offset: usize, + file: &dyn Storage, + sb_offset: u64, ndblks: usize, dblk_nelmts: usize, header: &ExtensibleArrayHeader, @@ -630,9 +655,9 @@ fn read_super_block( ) -> Result, FormatError> { let os = offset_size as usize; let sb_header_size = 4 + 1 + 1 + os + arr_off_size(header); - ensure_len(file_data, sb_offset, sb_header_size)?; + let prefix = read_exact_at(file, sb_offset, sb_header_size)?; - if &file_data[sb_offset..sb_offset + 4] != b"EASB" { + if &prefix[0..4] != b"EASB" { return Err(FormatError::ChunkedReadError( "invalid Extensible Array super block signature".into(), )); @@ -654,9 +679,19 @@ fn read_super_block( let bitmap_bytes = per_dblk_bitmap .checked_mul(ndblks) .ok_or_else(|| FormatError::Overflow("Extensible Array page bitmap size".into()))?; - let bitmap_start = sb_offset + sb_header_size; - ensure_len(file_data, bitmap_start, bitmap_bytes)?; - let bitmap = &file_data[bitmap_start..bitmap_start + bitmap_bytes]; + // Positions below are relative to the super block, whose bytes (up to + // its checksum) are all in one window. + let bitmap_start = sb_header_size; + let w = Window::read( + file, + sb_offset, + bitmap_start + .saturating_add(bitmap_bytes) + .saturating_add(ndblks.saturating_mul(os)) + .saturating_add(4), + )?; + w.ensure(bitmap_start, bitmap_bytes)?; + let bitmap = &w.bytes[bitmap_start..bitmap_start + bitmap_bytes]; let mut pos = bitmap_start + bitmap_bytes; let mut chunks = Vec::new(); @@ -667,16 +702,16 @@ fn read_super_block( .checked_mul(os) .and_then(|b| pos.checked_add(b)) .ok_or_else(|| FormatError::Overflow("Extensible Array super block span".into()))?; - verify_checksum(file_data, sb_offset, sb_end)?; + verify_checksum(&w, 0, sb_end)?; for i in 0..ndblks { - ensure_len(file_data, pos, os)?; - let addr = read_offset(file_data, pos, offset_size)?; + w.ensure(pos, os)?; + let addr = read_offset(&w.bytes, pos, offset_size)?; pos += os; if !is_undefined_addr(addr, offset_size) { chunks.extend(read_data_block_elements( - file_data, - addr as usize, + file, + addr, dblk_nelmts, header, offset_size, @@ -887,11 +922,11 @@ mod tests { assert_eq!(chunks[1].offsets, vec![20]); } - /// Build a synthetic EA with inline elements + one direct data block. - #[test] - fn read_inline_plus_data_blocks() { + /// A synthetic EA with inline elements + one direct data block: the + /// file, with the header at 0x100 (8-byte offsets and lengths, 4 chunks + /// of 10 elements from 0x1000 on). + fn build_inline_plus_data_blocks() -> Vec { let os: u8 = 8; - let ls: u8 = 8; let osv = os as usize; let chunk_byte_size = 10u64 * 8; // 10 elements × 8 bytes let idx_blk_elmts = 2u8; @@ -981,8 +1016,17 @@ mod tests { dbpos += osv; } stamp_checksum(&mut file_data, aedb_offset, dbpos); + file_data + } - let header = ExtensibleArrayHeader::parse(&file_data, aehd_offset, os, ls).unwrap(); + /// Build a synthetic EA with inline elements + one direct data block. + #[test] + fn read_inline_plus_data_blocks() { + let (os, ls) = (8u8, 8u8); + let chunk_byte_size = 10u64 * 8; + let base_addr = 0x1000u64; + let file_data = build_inline_plus_data_blocks(); + let header = ExtensibleArrayHeader::parse(&file_data, 0x100, os, ls).unwrap(); let ds_dims = vec![40u64]; let chunk_dims = vec![10u32]; let chunks = read_extensible_array_chunks( @@ -1018,7 +1062,8 @@ mod tests { fn read_element_unallocated() { let data = vec![0xFFu8; 16]; let grid = ChunkGrid::fixed_array(&[50], None, &[10]).unwrap(); - let (info, consumed) = read_element(&data, 0, 0, 8, 8, 80, 0, &grid).unwrap(); + let (info, consumed) = + read_element(&Window::whole(&data), 0, 0, 8, 8, 80, 0, &grid).unwrap(); assert!(info.is_none()); assert_eq!(consumed, 8); } @@ -1038,8 +1083,17 @@ mod tests { data[12..16].copy_from_slice(&0u32.to_le_bytes()); let grid = ChunkGrid::fixed_array(&[50], None, &[10]).unwrap(); - let (info, consumed) = - read_element(&data, 0, 1, elem_size as u8, os, 80, 2, &grid).unwrap(); + let (info, consumed) = read_element( + &Window::whole(&data), + 0, + 1, + elem_size as u8, + os, + 80, + 2, + &grid, + ) + .unwrap(); let ci = info.unwrap(); assert_eq!(ci.address, 0x2000); assert_eq!(ci.chunk_size, 120); @@ -1047,4 +1101,38 @@ mod tests { assert_eq!(ci.offsets, vec![20]); assert_eq!(consumed, elem_size); } + + /// The Storage path reads exactly what the slice path reads: the array + /// whole, cut at every length through its structures, and with a byte + /// damaged in each of them, through a read_at-only CountingStorage. + #[test] + fn storage_reads_match_slice_reads() { + use crate::storage::CountingStorage; + let full = build_inline_plus_data_blocks(); + let mut files = Vec::new(); + for cut in 0x100..0x340 { + files.push(full[..cut].to_vec()); + } + for at in [0x104, 0x150, 0x204, 0x216, 0x230, 0x304, 0x318] { + let mut damaged = full.clone(); + damaged[at] ^= 1; + files.push(damaged); + } + files.push(full); + let mut compared = 0; + for f in files { + let storage = CountingStorage::new(f.clone()); + let want = ExtensibleArrayHeader::parse(&f, 0x100, 8, 8); + let got = ExtensibleArrayHeader::parse_in(&storage, 0x100, 8, 8); + assert_eq!(format!("{got:?}"), format!("{want:?}")); + let Ok(h) = want else { continue }; + for dims in [&[40u64][..], &[25]] { + let want = read_extensible_array_chunks(&f, &h, dims, None, &[10], 8, 8, 8); + let got = read_extensible_array_chunks_in(&storage, &h, dims, None, &[10], 8, 8, 8); + assert_eq!(format!("{got:?}"), format!("{want:?}"), "{} bytes", f.len()); + compared += 1; + } + } + assert!(compared > 100); + } } diff --git a/crates/clawhdf5-format/src/storage.rs b/crates/clawhdf5-format/src/storage.rs index 4608b2d..c0a1c37 100644 --- a/crates/clawhdf5-format/src/storage.rs +++ b/crates/clawhdf5-format/src/storage.rs @@ -240,6 +240,16 @@ impl<'a> Window<'a> { }) } + /// A whole in-memory file as one window (base 0). + #[cfg(test)] + pub fn whole(bytes: &'a [u8]) -> Self { + Window { + bytes: Cow::Borrowed(bytes), + base: 0, + file_len: bytes.len(), + } + } + /// Check that `[rel, rel + needed)` (relative to `base`) is in the file. pub fn ensure(&self, rel: usize, needed: usize) -> Result<(), FormatError> { match rel.checked_add(needed) {