diff --git a/crates/clawhdf5-format/src/addr.rs b/crates/clawhdf5-format/src/addr.rs index ec25e3d..09c2cef 100644 --- a/crates/clawhdf5-format/src/addr.rs +++ b/crates/clawhdf5-format/src/addr.rs @@ -23,6 +23,19 @@ pub fn to_usize(value: u64) -> Result { usize::try_from(value).map_err(|_| too_large(value)) } +/// A count or offset into an in-memory buffer (a codec's progress counter, +/// a size the writer computed from data it holds) as a `usize`, saturating +/// at `usize::MAX` instead of truncating. +/// +/// For values that are bounded by the length of something in memory, so +/// always fit; if one ever did not, a saturated index fails its bounds check +/// or allocation instead of silently addressing the wrong bytes. A value +/// read from the file uses [`to_usize`]. +#[inline] +pub fn saturating_usize(value: u64) -> usize { + usize::try_from(value).unwrap_or(usize::MAX) +} + #[cold] #[inline(never)] fn too_large(value: u64) -> FormatError { @@ -42,6 +55,15 @@ mod tests { assert_eq!(to_usize(usize::MAX as u64), Ok(usize::MAX)); } + #[test] + fn saturating_conversion_never_wraps() { + assert_eq!(saturating_usize(0), 0); + assert_eq!(saturating_usize(0x1234), 0x1234); + assert_eq!(saturating_usize(usize::MAX as u64), usize::MAX); + // Past usize::MAX (32-bit targets) or at u64::MAX: saturates. + assert_eq!(saturating_usize(u64::MAX), usize::MAX); + } + #[test] fn values_past_usize_max_are_an_error_not_truncated() { // Only reachable where usize is narrower than u64; on a 64-bit host diff --git a/crates/clawhdf5-format/src/btree_v1.rs b/crates/clawhdf5-format/src/btree_v1.rs index b652dcc..0e9b7fe 100644 --- a/crates/clawhdf5-format/src/btree_v1.rs +++ b/crates/clawhdf5-format/src/btree_v1.rs @@ -3,6 +3,7 @@ #[cfg(not(feature = "std"))] use alloc::vec::Vec; +use crate::addr::to_usize; use crate::error::FormatError; /// A parsed B-tree v1 node. @@ -164,7 +165,12 @@ fn collect_symbol_table_nodes_inner( return Err(FormatError::NestingDepthExceeded); } - let node = BTreeV1Node::parse(file_data, btree_address as usize, offset_size, length_size)?; + let node = BTreeV1Node::parse( + file_data, + to_usize(btree_address)?, + offset_size, + length_size, + )?; if node.node_type != 0 { return Err(FormatError::InvalidBTreeNodeType(node.node_type)); diff --git a/crates/clawhdf5-format/src/btree_v2_write.rs b/crates/clawhdf5-format/src/btree_v2_write.rs index 795fb80..38fa7c1 100644 --- a/crates/clawhdf5-format/src/btree_v2_write.rs +++ b/crates/clawhdf5-format/src/btree_v2_write.rs @@ -6,6 +6,7 @@ //! libhdf5 uses (`H5B2__hdr_init`) and the reader decodes pointers with, so //! the pointer widths the writer encodes are the ones every reader expects. +use crate::addr::saturating_usize; #[cfg(not(feature = "std"))] use alloc::{format, vec, vec::Vec}; @@ -105,7 +106,9 @@ pub(crate) fn build_btree_v2( first_node: addr + hdr_len as u64, nodes: Vec::new(), }; - let root = (n > 0).then(|| w.node(depth, 0, n as usize)).transpose()?; + let root = (n > 0) + .then(|| w.node(depth, 0, saturating_usize(n))) + .transpose()?; let mut out = Vec::with_capacity(hdr_len + w.nodes.len() * p.node_size as usize); out.extend_from_slice(b"BTHD"); @@ -201,7 +204,7 @@ impl TreeWriter<'_> { "cannot spread {n} B-tree v2 records over {k} children at depth {depth}" ))); } - let k = k as usize; + let k = saturating_usize(k); let in_children = n - (k - 1); let (base, extra) = (in_children / k, in_children % k); diff --git a/crates/clawhdf5-format/src/chunk_index.rs b/crates/clawhdf5-format/src/chunk_index.rs index c307572..706bad0 100644 --- a/crates/clawhdf5-format/src/chunk_index.rs +++ b/crates/clawhdf5-format/src/chunk_index.rs @@ -18,6 +18,7 @@ use alloc::collections::BTreeMap; #[cfg(feature = "std")] use std::collections::HashMap; +use crate::addr::to_usize; use crate::chunk_cache::ChunkCoord; use crate::chunked_read::ChunkInfo; @@ -167,7 +168,15 @@ impl ChunkLayout { for (_coord, ci) in index.iter() { let coord: ChunkCoord = ci.offsets.iter().take(rank).copied().collect(); - let chunk_offsets: Vec = coord.iter().map(|&o| o as usize).collect(); + // `ds_dims` are `usize`: a chunk at an offset past `usize::MAX` + // (only on a 32-bit target) lies outside the dataset. + let Ok(chunk_offsets) = coord + .iter() + .map(|&o| to_usize(o)) + .collect::, _>>() + else { + continue; + }; let copies = if rank == 0 { // Scalar dataset — single copy diff --git a/crates/clawhdf5-format/src/chunked_read.rs b/crates/clawhdf5-format/src/chunked_read.rs index 6dda3cb..e249168 100644 --- a/crates/clawhdf5-format/src/chunked_read.rs +++ b/crates/clawhdf5-format/src/chunked_read.rs @@ -6,6 +6,7 @@ extern crate alloc; #[cfg(not(feature = "std"))] use alloc::{format, vec, vec::Vec}; +use crate::addr::to_usize; #[cfg(feature = "std")] use crate::chunk_cache::{CacheAlignedBuffer, ChunkCache}; use crate::data_layout::DataLayout; @@ -265,7 +266,7 @@ fn fill_from_chunks( ))); } let offsets = &c.offsets[..rank]; - let c_addr = c.address as usize; + let c_addr = to_usize(c.address)?; let size = c.chunk_size as usize; ensure_len(file_data, c_addr, size)?; let raw = &file_data[c_addr..c_addr + size]; @@ -825,7 +826,7 @@ fn parse_chunk_node( return Err(FormatError::NestingDepthExceeded); } - let offset = btree_address as usize; + let offset = to_usize(btree_address)?; let os = offset_size as usize; // Parse B-tree v1 header @@ -945,7 +946,8 @@ pub fn generate_implicit_chunks( } let total_chunks: u64 = num_chunks_per_dim.iter().product(); - let mut chunks = Vec::with_capacity(total_chunks as usize); + // A capacity hint only (a count past `usize::MAX` could not be pushed). + let mut chunks = Vec::with_capacity(usize::try_from(total_chunks).unwrap_or(0)); for linear_idx in 0..total_chunks { let mut offsets = vec![0u64; rank]; let mut remaining = linear_idx; @@ -993,7 +995,7 @@ fn read_btree_v2_chunks( use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records}; let bad = |what: &str| FormatError::ChunkedReadError(format!("B-tree v2 chunk index: {what}")); - let header = BTreeV2Header::parse(file_data, addr as usize, offset_size, length_size)?; + let header = BTreeV2Header::parse(file_data, to_usize(addr)?, offset_size, length_size)?; let rank = chunk_dims.len(); let os = offset_size as usize; let record_size = header.record_size as usize; @@ -1122,7 +1124,11 @@ pub fn list_chunks( // Both v3 and v4 include element size as last dim (rank+1) let (rank, chunk_dims) = chunk_geometry(chunk_dimensions, version, dataspace, elem_size)?; - let ds_dims: Vec = dataspace.dimensions.iter().map(|&d| d as usize).collect(); + let ds_dims: Vec = dataspace + .dimensions + .iter() + .map(|&d| to_usize(d)) + .collect::>()?; // Collect chunks based on version and index type let mut chunks = match (version, chunk_index_type) { @@ -1158,7 +1164,7 @@ pub fn list_chunks( // Fixed Array — use spatial chunk dims only let spatial_chunk_dims: &[u32] = &chunk_dimensions[..rank]; let header = - FixedArrayHeader::parse(file_data, addr as usize, offset_size, length_size)?; + FixedArrayHeader::parse(file_data, to_usize(addr)?, offset_size, length_size)?; read_fixed_array_chunks( file_data, &header, @@ -1174,7 +1180,7 @@ pub fn list_chunks( // Extensible Array — use spatial chunk dims only let spatial_chunk_dims: &[u32] = &chunk_dimensions[..rank]; let header = - ExtensibleArrayHeader::parse(file_data, addr as usize, offset_size, length_size)?; + ExtensibleArrayHeader::parse(file_data, to_usize(addr)?, offset_size, length_size)?; read_extensible_array_chunks( file_data, &header, @@ -1349,7 +1355,11 @@ pub(crate) fn read_chunked_full( // dimension the total is 0 even if other dimensions are huge. return Ok(output); } - let ds_dims: Vec = dataspace.dimensions.iter().map(|&d| d as usize).collect(); + let ds_dims: Vec = dataspace + .dimensions + .iter() + .map(|&d| to_usize(d)) + .collect::>()?; let placer = ChunkPlacer::new(&chunk_dims, &ds_dims, elem_size); let chunk_total_bytes = checked_chunk_byte_len(&chunk_dims, elem_size)?; // Chunks are cached only when the whole dataset fits: pushing a larger @@ -1603,7 +1613,11 @@ pub fn read_chunked_data_sweep( check_chunk_element_size(layout, datatype, offset_size)?; let elem_size = datatype.type_size() as usize; let (rank, chunk_dims) = chunk_geometry(chunk_dimensions, version, dataspace, elem_size)?; - let ds_dims: Vec = dataspace.dimensions.iter().map(|&d| d as usize).collect(); + let ds_dims: Vec = dataspace + .dimensions + .iter() + .map(|&d| to_usize(d)) + .collect::>()?; // The per-file cache is shared across datasets (and threads); every // lookup is keyed by this dataset's chunk-index address, so another @@ -1659,7 +1673,7 @@ pub fn read_chunked_data_sweep( cached } else { // Decompress from file - let c_addr = chunk_info.address as usize; + let c_addr = to_usize(chunk_info.address)?; let size = chunk_info.chunk_size as usize; ensure_len(file_data, c_addr, size)?; let raw_chunk = &file_data[c_addr..c_addr + size]; @@ -1682,8 +1696,8 @@ pub fn read_chunked_data_sweep( .offsets .iter() .take(rank) - .map(|&o| o as usize) - .collect(); + .map(|&o| to_usize(o)) + .collect::>()?; if rank == 0 { let copy_len = decompressed.len().min(output.len()); @@ -1743,7 +1757,11 @@ pub fn read_chunked_data_indexed( check_chunk_element_size(layout, datatype, offset_size)?; let elem_size = datatype.type_size() as usize; let (rank, chunk_dims) = chunk_geometry(chunk_dimensions, version, dataspace, elem_size)?; - let ds_dims: Vec = dataspace.dimensions.iter().map(|&d| d as usize).collect(); + let ds_dims: Vec = dataspace + .dimensions + .iter() + .map(|&d| to_usize(d)) + .collect::>()?; // Chunk index and assembly plan for this dataset, built on first access // and kept per dataset (keyed by chunk-index address) in the shared cache. @@ -1776,7 +1794,7 @@ pub fn read_chunked_data_indexed( if let Some(cached) = cache.get_decompressed_in(addr, coord) { chunk_buffers.push(cached); } else { - let c_addr = *file_offset as usize; + let c_addr = to_usize(*file_offset)?; let size = *file_size as usize; ensure_len(file_data, c_addr, size)?; let raw_chunk = &file_data[c_addr..c_addr + size]; diff --git a/crates/clawhdf5-format/src/chunked_write.rs b/crates/clawhdf5-format/src/chunked_write.rs index 2a8bd09..cf5c958 100644 --- a/crates/clawhdf5-format/src/chunked_write.rs +++ b/crates/clawhdf5-format/src/chunked_write.rs @@ -3,6 +3,7 @@ #[cfg(not(feature = "std"))] extern crate alloc; +use crate::addr::saturating_usize; #[cfg(not(feature = "std"))] use alloc::{format, vec, vec::Vec}; @@ -414,18 +415,18 @@ pub fn split_into_chunks( // Dataset strides (row-major) let mut ds_strides = vec![1usize; rank]; for i in (0..rank.saturating_sub(1)).rev() { - ds_strides[i] = ds_strides[i + 1] * shape[i + 1] as usize; + ds_strides[i] = ds_strides[i + 1] * saturating_usize(shape[i + 1]); } // Chunk strides let mut chunk_strides = vec![1usize; rank]; for i in (0..rank.saturating_sub(1)).rev() { - chunk_strides[i] = chunk_strides[i + 1] * chunk_dims[i + 1] as usize; + chunk_strides[i] = chunk_strides[i + 1] * saturating_usize(chunk_dims[i + 1]); } - let chunk_total_elements: usize = chunk_dims.iter().map(|&d| d as usize).product(); + let chunk_total_elements: usize = chunk_dims.iter().map(|&d| saturating_usize(d)).product(); - let mut result = Vec::with_capacity(total_chunks as usize); + let mut result = Vec::with_capacity(saturating_usize(total_chunks)); for linear_idx in 0..total_chunks { // Convert linear index to chunk grid coordinates @@ -453,8 +454,8 @@ pub fn split_into_chunks( let coord_in_chunk = remaining_idx / chunk_strides[d]; remaining_idx %= chunk_strides[d]; - let global_coord = offsets[d] as usize + coord_in_chunk; - if global_coord >= shape[d] as usize { + let global_coord = saturating_usize(offsets[d]) + coord_in_chunk; + if global_coord >= saturating_usize(shape[d]) { out_of_bounds = true; break; } @@ -1036,7 +1037,7 @@ impl ChunkIndexPlan { Ok(Self::SingleChunk) } else { let grid = ChunkGrid::fixed_array(shape, Some(max), chunk_dims)?; - Ok(Self::FixedArray(grid, nslots as usize)) + Ok(Self::FixedArray(grid, saturating_usize(nslots))) } } 1 => Ok(Self::ExtensibleArray(ChunkGrid::extensible_array( @@ -1251,7 +1252,7 @@ pub fn write_selection_to_buffer( let rank = dims.len(); let mut ds_strides = vec![1usize; rank]; for i in (0..rank.saturating_sub(1)).rev() { - ds_strides[i] = ds_strides[i + 1] * dims[i + 1] as usize; + ds_strides[i] = ds_strides[i + 1] * saturating_usize(dims[i + 1]); } let mut src_offset = 0usize; @@ -1301,7 +1302,7 @@ pub fn write_selection_to_buffer( buffer, new_data, src_offset, - current_ds_offset + coord as usize * ds_strides[d], + current_ds_offset + saturating_usize(coord) * ds_strides[d], ); } } @@ -1328,14 +1329,14 @@ pub fn write_selection_to_buffer( let rank = dims.len(); let mut ds_strides = vec![1usize; rank]; for i in (0..rank.saturating_sub(1)).rev() { - ds_strides[i] = ds_strides[i + 1] * dims[i + 1] as usize; + ds_strides[i] = ds_strides[i + 1] * saturating_usize(dims[i + 1]); } for (pi, pt) in pts.iter().enumerate() { let flat: usize = pt .iter() .zip(ds_strides.iter()) - .map(|(&p, &s)| p as usize * s) + .map(|(&p, &s)| saturating_usize(p) * s) .sum(); let dst = flat * elem_size; let src = pi * elem_size; diff --git a/crates/clawhdf5-format/src/data_layout.rs b/crates/clawhdf5-format/src/data_layout.rs index 59a9065..c9b93c7 100644 --- a/crates/clawhdf5-format/src/data_layout.rs +++ b/crates/clawhdf5-format/src/data_layout.rs @@ -6,6 +6,7 @@ use alloc::{format, string::String, vec::Vec}; #[cfg(feature = "std")] use std::string::String; +use crate::addr::to_usize; use crate::error::FormatError; /// A single VDS (Virtual Dataset) source mapping. @@ -207,7 +208,7 @@ pub fn parse_vds_mappings( "VDS mapping shares a name with a later entry".into(), )); } - Ok(idx as usize) + to_usize(idx) }; let source_file = if flags & VDS_SOURCE_SAME_FILE != 0 { @@ -320,7 +321,7 @@ impl DataLayout { { let coll = crate::global_heap::GlobalHeapCollection::parse( file_data, - addr as usize, + to_usize(addr)?, length_size, )?; let obj = coll.get_object(*global_heap_index as u16).ok_or( diff --git a/crates/clawhdf5-format/src/data_read.rs b/crates/clawhdf5-format/src/data_read.rs index ef73a04..bcf1a18 100644 --- a/crates/clawhdf5-format/src/data_read.rs +++ b/crates/clawhdf5-format/src/data_read.rs @@ -6,6 +6,7 @@ use alloc::{collections::BTreeMap, format, string::String, vec, vec::Vec}; #[cfg(feature = "std")] use std::collections::BTreeMap; +use crate::addr::to_usize; #[cfg(feature = "std")] use crate::chunk_cache::ChunkCache; use crate::chunked_read::read_chunked_data; @@ -117,7 +118,7 @@ pub fn read_raw_data_zerocopy<'a>( dataspace: &Dataspace, datatype: &Datatype, ) -> Result, FormatError> { - let num_elements = dataspace.num_elements() as usize; + let num_elements = to_usize(dataspace.num_elements())?; let elem_size = datatype.type_size() as usize; let expected_size = num_elements.checked_mul(elem_size).ok_or_else(|| { FormatError::Overflow(format!( @@ -128,7 +129,7 @@ pub fn read_raw_data_zerocopy<'a>( match layout { DataLayout::Contiguous { address, size } => { let addr = address.ok_or(FormatError::NoDataAllocated)?; - let addr = addr as usize; + let addr = to_usize(addr)?; let sz = contiguous_read_len(*size, expected_size)?; ensure_len(file_data, addr, sz)?; Ok(Some(&file_data[addr..addr + sz])) @@ -219,7 +220,7 @@ fn read_raw_data_full_impl( length_size: u8, resolver: Option<&VdsSourceResolver>, ) -> Result, FormatError> { - let num_elements = dataspace.num_elements() as usize; + let num_elements = to_usize(dataspace.num_elements())?; let elem_size = datatype.type_size() as usize; let expected_size = num_elements.checked_mul(elem_size).ok_or_else(|| { FormatError::Overflow(format!( @@ -239,7 +240,7 @@ fn read_raw_data_full_impl( } DataLayout::Contiguous { address, size } => { let addr = address.ok_or(FormatError::NoDataAllocated)?; - let addr = addr as usize; + let addr = to_usize(addr)?; let sz = contiguous_read_len(*size, expected_size)?; ensure_len(file_data, addr, sz)?; let mut out = crate::bulk_alloc::vec_for_bulk(sz); @@ -582,7 +583,7 @@ pub fn extract_selection_from_buffer( let rank = dims.len(); let mut ds_strides = vec![1usize; rank]; for i in (0..rank.saturating_sub(1)).rev() { - ds_strides[i] = ds_strides[i + 1] * dims[i + 1] as usize; + ds_strides[i] = ds_strides[i + 1] * to_usize(dims[i + 1])?; } let mut output = Vec::with_capacity(pts.len() * elem_size); @@ -590,8 +591,8 @@ pub fn extract_selection_from_buffer( let flat: usize = pt .iter() .zip(ds_strides.iter()) - .map(|(&p, &s)| p as usize * s) - .sum(); + .map(|(&p, &s)| Ok(to_usize(p)? * s)) + .sum::>()?; let src = flat * elem_size; if src + elem_size <= full_data.len() { output.extend_from_slice(&full_data[src..src + elem_size]); @@ -1341,7 +1342,7 @@ pub fn read_compound_fields( let mut fields = Vec::with_capacity(members.len()); for m in members { let field_size = m.datatype.type_size() as usize; - let offset = m.byte_offset as usize; + let offset = to_usize(m.byte_offset)?; if offset .checked_add(field_size) .is_none_or(|end| end > elem_size) diff --git a/crates/clawhdf5-format/src/ea_writer.rs b/crates/clawhdf5-format/src/ea_writer.rs index f669534..de7859c 100644 --- a/crates/clawhdf5-format/src/ea_writer.rs +++ b/crates/clawhdf5-format/src/ea_writer.rs @@ -3,6 +3,7 @@ #[cfg(not(feature = "std"))] extern crate alloc; +use crate::addr::saturating_usize; #[cfg(not(feature = "std"))] use alloc::{vec, vec::Vec}; @@ -247,7 +248,7 @@ pub fn build_extensible_array_at( // Header (EAHD). The six statistics are, in order: super blocks, their // bytes, data blocks, their bytes, max index set, elements realised. - let mut out = Vec::with_capacity((cursor - ea_base_address) as usize); + let mut out = Vec::with_capacity(saturating_usize(cursor - ea_base_address)); out.extend_from_slice(b"EAHD"); out.push(0); // version out.push(client_id); diff --git a/crates/clawhdf5-format/src/extensible_array.rs b/crates/clawhdf5-format/src/extensible_array.rs index ba1c822..9256326 100644 --- a/crates/clawhdf5-format/src/extensible_array.rs +++ b/crates/clawhdf5-format/src/extensible_array.rs @@ -9,6 +9,7 @@ extern crate alloc; #[cfg(not(feature = "std"))] use alloc::{format, vec, vec::Vec}; +use crate::addr::to_usize; use crate::chunk_grid::ChunkGrid; use crate::chunked_read::ChunkInfo; use crate::error::FormatError; @@ -451,7 +452,7 @@ pub fn read_extensible_array_chunks( // Parse index block (EAIB): signature(4) + version(1) + client_id(1) // + header address(offset_size), then the inline elements, then the // direct data block addresses, then the super block addresses. - let ib_offset = header.index_block_address as usize; + let ib_offset = to_usize(header.index_block_address)?; let ib_header_size = 4 + 1 + 1 + os; ensure_len(file_data, ib_offset, ib_header_size)?; @@ -463,7 +464,7 @@ pub fn read_extensible_array_chunks( let mut pos = ib_offset + ib_header_size; let mut chunks = Vec::new(); - let total_elements = header.num_elements as usize; + let total_elements = to_usize(header.num_elements)?; let dmin = header.min_dblk_nelmts as usize; if dmin == 0 || !dmin.is_power_of_two() { @@ -563,7 +564,7 @@ pub fn read_extensible_array_chunks( } chunks.extend(read_data_block_elements( file_data, - addr as usize, + to_usize(addr)?, dblk_nelmts, header, offset_size, @@ -592,7 +593,7 @@ pub fn read_extensible_array_chunks( if !is_undefined_addr(sb_addr, offset_size) { chunks.extend(read_super_block( file_data, - sb_addr as usize, + to_usize(sb_addr)?, ndblks, dblk_nelmts, header, @@ -676,7 +677,7 @@ fn read_super_block( if !is_undefined_addr(addr, offset_size) { chunks.extend(read_data_block_elements( file_data, - addr as usize, + to_usize(addr)?, dblk_nelmts, header, offset_size, diff --git a/crates/clawhdf5-format/src/file_writer.rs b/crates/clawhdf5-format/src/file_writer.rs index 2568582..1a1af83 100644 --- a/crates/clawhdf5-format/src/file_writer.rs +++ b/crates/clawhdf5-format/src/file_writer.rs @@ -3,6 +3,7 @@ //! Produces valid HDF5 files with v3 superblock, v2 object headers, //! link messages, contiguous datasets, inline and dense attributes. +use crate::addr::saturating_usize; #[cfg(not(feature = "std"))] use alloc::{format, vec, vec::Vec}; @@ -336,7 +337,7 @@ pub(crate) fn build_single_block_fractal_heap( // An object must fit one direct block: the writer has no huge-object // path, and libhdf5 cannot read an object that overruns its block. - let max_managed = max_direct_block_size as usize - dblock_header_size; + let max_managed = saturating_usize(max_direct_block_size) - dblock_header_size; if let Some(big) = serialized.iter().find(|s| s.len() > max_managed) { return Err(FormatError::SerializationError(format!( "a {}-byte message cannot go in dense storage: a fractal heap \ @@ -392,7 +393,7 @@ pub(crate) fn build_single_block_fractal_heap( let dblock_addr = frhp_addr + frhp_size as u64; let btree_addr = dblock_addr + starting_block_size; - let data_space = starting_block_size as usize - dblock_header_size; + let data_space = saturating_usize(starting_block_size) - dblock_header_size; let free_space = data_space - total_data_size; // Build fractal heap header @@ -428,7 +429,7 @@ pub(crate) fn build_single_block_fractal_heap( debug_assert_eq!(frhp.len(), frhp_size); // Build direct block: header (with checksum) + data + padding - let mut dblock = Vec::with_capacity(starting_block_size as usize); + let mut dblock = Vec::with_capacity(saturating_usize(starting_block_size)); dblock.extend_from_slice(b"FHDB"); dblock.push(0); // version write_offset(&mut dblock, frhp_addr, OFFSET_SIZE); @@ -446,12 +447,12 @@ pub(crate) fn build_single_block_fractal_heap( } // Pad to full block size - dblock.resize(starting_block_size as usize, 0); + dblock.resize(saturating_usize(starting_block_size), 0); // Checksum: computed over entire block with checksum field zeroed let dblock_checksum = crate::checksum::jenkins_lookup3(&dblock); dblock[cksum_pos..cksum_pos + 4].copy_from_slice(&dblock_checksum.to_le_bytes()); - debug_assert_eq!(dblock.len(), starting_block_size as usize); + debug_assert_eq!(dblock.len(), saturating_usize(starting_block_size)); // Build heap IDs let heap_ids: Vec> = obj_offsets @@ -706,7 +707,7 @@ impl HeapIndirectBlock { let cksum_pos = out.len(); out.extend_from_slice(&[0u8; 4]); // checksum placeholder out.extend_from_slice(&b.data); - out.resize(d + b.size as usize, 0); + out.resize(d + saturating_usize(b.size), 0); let cksum = crate::checksum::jenkins_lookup3(&out[d..]); out[cksum_pos..cksum_pos + 4].copy_from_slice(&cksum.to_le_bytes()); child += b.size; @@ -740,7 +741,7 @@ impl HeapPacker<'_> { nrows: Option, ) -> Result { let geom = self.geom; - let width = geom.width as usize; + let width = saturating_usize(geom.width); let mut slots = Vec::new(); let mut off = heap_offset; let mut row = 0usize; @@ -763,7 +764,8 @@ impl HeapPacker<'_> { // A child whose biggest direct block cannot hold the // next object is skipped whole, not walked. let biggest = geom.row_size(child_rows.min(geom.max_direct_rows()) - 1); - if self.objects[self.next].len() > (biggest as usize - geom.dblock_header_size) + if self.objects[self.next].len() + > (saturating_usize(biggest) - geom.dblock_header_size) { slots.push(HeapSlot::Empty); off += size; @@ -794,7 +796,7 @@ impl HeapPacker<'_> { /// objects as fit; leave it unallocated if not even the next one does. fn fill_direct(&mut self, heap_offset: u64, size: u64) -> HeapSlot { let header = self.geom.dblock_header_size; - let capacity = size as usize - header; + let capacity = saturating_usize(size) - header; let mut data = Vec::new(); while let Some(obj) = self.objects.get(self.next) { if data.len() + obj.len() > capacity { diff --git a/crates/clawhdf5-format/src/fill_value.rs b/crates/clawhdf5-format/src/fill_value.rs index 3b18790..d8c34f3 100644 --- a/crates/clawhdf5-format/src/fill_value.rs +++ b/crates/clawhdf5-format/src/fill_value.rs @@ -12,6 +12,7 @@ #[cfg(not(feature = "std"))] use alloc::{format, vec, vec::Vec}; +use crate::addr::to_usize; use crate::chunked_read::{alloc_output, checked_byte_len, list_chunks}; use crate::data_layout::DataLayout; use crate::dataspace::Dataspace; @@ -256,7 +257,11 @@ pub fn apply_to_unallocated_chunks( length_size, )?; let rank = chunk_dims.len(); - let ds_dims: Vec = dataspace.dimensions.iter().map(|&d| d as usize).collect(); + let ds_dims: Vec = dataspace + .dimensions + .iter() + .map(|&d| to_usize(d)) + .collect::>()?; if rank == 0 || ds_dims.len() != rank || chunk_dims.contains(&0) { return Ok(()); } @@ -288,7 +293,7 @@ pub fn apply_to_unallocated_chunks( let mut cell = 0usize; let mut in_range = true; for d in 0..rank { - let coord = chunk.offsets[d] as usize / chunk_dims[d]; + let coord = to_usize(chunk.offsets[d])? / chunk_dims[d]; if coord >= grid[d] { in_range = false; break; diff --git a/crates/clawhdf5-format/src/filters.rs b/crates/clawhdf5-format/src/filters.rs index 40d6cd9..672649b 100644 --- a/crates/clawhdf5-format/src/filters.rs +++ b/crates/clawhdf5-format/src/filters.rs @@ -3,6 +3,8 @@ #[cfg(not(feature = "std"))] extern crate alloc; +#[cfg(feature = "deflate")] +use crate::addr::saturating_usize; #[cfg(not(feature = "std"))] use alloc::{boxed::Box, format, vec, vec::Vec}; @@ -1114,7 +1116,11 @@ fn inflate_bounded_into( loop { let (in_before, out_before) = (inflater.total_in(), inflater.total_out()); let status = inflater - .decompress_vec(&data[in_before as usize..], out, FlushDecompress::Finish) + .decompress_vec( + &data[saturating_usize(in_before)..], + out, + FlushDecompress::Finish, + ) .map_err(|e| format!("deflate: {e}"))?; if out.len() > limit { return Err("deflate: output exceeds size limit".into()); @@ -1132,7 +1138,7 @@ fn inflate_bounded_into( } Status::Ok | Status::BufError => { // Room left, so the decoder stopped for want of input. - if inflater.total_in() as usize >= data.len() + if saturating_usize(inflater.total_in()) >= data.len() || (inflater.total_in(), inflater.total_out()) == (in_before, out_before) { return Err("deflate: truncated stream".into()); @@ -1232,7 +1238,11 @@ pub(crate) fn deflate_bounded(data: &[u8], level: u32) -> Result, String loop { let (in_before, out_before) = (deflater.total_in(), deflater.total_out()); let status = deflater - .compress_vec(&data[in_before as usize..], &mut out, FlushCompress::Finish) + .compress_vec( + &data[saturating_usize(in_before)..], + &mut out, + FlushCompress::Finish, + ) .map_err(|e| format!("deflate: {e}"))?; match status { Status::StreamEnd => return Ok(out), diff --git a/crates/clawhdf5-format/src/filters_blosc2.rs b/crates/clawhdf5-format/src/filters_blosc2.rs index f4d1690..a609a26 100644 --- a/crates/clawhdf5-format/src/filters_blosc2.rs +++ b/crates/clawhdf5-format/src/filters_blosc2.rs @@ -46,6 +46,7 @@ //! variable-length blocks, dictionaries, lazy chunks, user-defined codecs //! and registered filters (e.g. bytedelta), sparse frames. +use crate::addr::saturating_usize; use crate::error::FormatError; use crate::filter_registry::FilterContext; use crate::filters_bitshuffle::bitunshuffle_block; @@ -546,7 +547,7 @@ fn parse_frame(buf: &[u8], limit: usize) -> Result, FormatError> { return Err(err("negative size in frame header")); } let header_len = header_len as usize; - let buf = &buf[..frame_len as usize]; + let buf = &buf[..saturating_usize(frame_len)]; let cbytes = usize::try_from(cbytes).map_err(|_| err("bad compressed size"))?; let data_end = header_len .checked_add(cbytes) diff --git a/crates/clawhdf5-format/src/filters_bzip2.rs b/crates/clawhdf5-format/src/filters_bzip2.rs index 93e33a3..6989c5b 100644 --- a/crates/clawhdf5-format/src/filters_bzip2.rs +++ b/crates/clawhdf5-format/src/filters_bzip2.rs @@ -4,6 +4,7 @@ //! the compression level). Decoded with the `bzip2` crate's default backend, //! `libbz2-rs-sys`, a pure-Rust port of libbzip2. +use crate::addr::saturating_usize; use crate::error::FormatError; use crate::filter_registry::FilterContext; @@ -28,7 +29,7 @@ pub(crate) fn bzip2_decode(input: &[u8], ctx: &FilterContext<'_>) -> Result limit { return Err(err("output exceeds the chunk size")); @@ -43,7 +44,7 @@ pub(crate) fn bzip2_decode(input: &[u8], ctx: &FilterContext<'_>) -> Result= input.len() + } else if saturating_usize(dec.total_in()) >= input.len() || (dec.total_in(), dec.total_out()) == (in_before, out_before) { return Err(err("truncated stream")); @@ -61,7 +62,7 @@ pub(crate) fn bzip2_encode(input: &[u8], ctx: &FilterContext<'_>) -> Result Result, FormatError> { - let db_offset = header.data_block_address as usize; + let db_offset = to_usize(header.data_block_address)?; // Parse data block header: FADB(4) + version(1) + client_id(1) + header_address(offset_size) let db_header_size = 4 + 1 + 1 + offset_size as usize; @@ -174,7 +175,7 @@ pub fn read_fixed_array_chunks( // Elements start immediately after the data block prefix. let elements_start = db_offset + db_header_size; - let num_elements = header.num_elements as usize; + let num_elements = to_usize(header.num_elements)?; // A chunk index cannot describe more elements than the file has bytes (each // element occupies at least `offset_size` bytes). Reject a corrupt count // before it can drive a huge loop or overflow an offset computation. diff --git a/crates/clawhdf5-format/src/group_v1.rs b/crates/clawhdf5-format/src/group_v1.rs index a4f97c8..3dff9cd 100644 --- a/crates/clawhdf5-format/src/group_v1.rs +++ b/crates/clawhdf5-format/src/group_v1.rs @@ -3,6 +3,7 @@ #[cfg(not(feature = "std"))] use alloc::{string::String, vec::Vec}; +use crate::addr::to_usize; use crate::btree_v1::collect_symbol_table_nodes; use crate::error::FormatError; use crate::local_heap::LocalHeap; @@ -54,7 +55,7 @@ pub(crate) fn v1_group_entries( // Parse local heap let heap = LocalHeap::parse( file_data, - sym_table_msg.local_heap_address as usize, + to_usize(sym_table_msg.local_heap_address)?, offset_size, length_size, )?; @@ -70,7 +71,7 @@ pub(crate) fn v1_group_entries( let mut entries = Vec::new(); let mut heap_checked = false; for snod_addr in snod_addrs { - let snod = SymbolTableNode::parse(file_data, snod_addr as usize, offset_size)?; + let snod = SymbolTableNode::parse(file_data, to_usize(snod_addr)?, offset_size)?; for entry in &snod.entries { // Like libhdf5, look at the heap's free list only once a name is // needed: an empty group with a damaged heap still lists. @@ -152,7 +153,7 @@ fn for_each_v1_soft_link( ) -> Result<(), FormatError> { let heap = LocalHeap::parse( file_data, - sym_table_msg.local_heap_address as usize, + to_usize(sym_table_msg.local_heap_address)?, offset_size, length_size, )?; @@ -164,7 +165,7 @@ fn for_each_v1_soft_link( )?; let mut heap_checked = false; for snod_addr in snod_addrs { - let snod = SymbolTableNode::parse(file_data, snod_addr as usize, offset_size)?; + let snod = SymbolTableNode::parse(file_data, to_usize(snod_addr)?, offset_size)?; for entry in &snod.entries { if entry.cache_type != CACHE_TYPE_SOFT_LINK { continue; @@ -242,7 +243,7 @@ pub fn resolve_path( // Not last — must be a group, parse its object header to get symbol table let obj_header = ObjectHeader::parse( file_data, - entry.object_header_address as usize, + to_usize(entry.object_header_address)?, offset_size, length_size, )?; diff --git a/crates/clawhdf5-format/src/link_message.rs b/crates/clawhdf5-format/src/link_message.rs index 27044cc..55cb269 100644 --- a/crates/clawhdf5-format/src/link_message.rs +++ b/crates/clawhdf5-format/src/link_message.rs @@ -3,6 +3,7 @@ #[cfg(not(feature = "std"))] use alloc::{string::String, vec::Vec}; +use crate::addr::to_usize; use crate::datatype::CharacterSet; use crate::error::FormatError; @@ -247,7 +248,7 @@ impl LinkMessage { }; // Link name length - let name_len = read_offset(data, pos, name_size_field_width)? as usize; + let name_len = to_usize(read_offset(data, pos, name_size_field_width)?)?; pos += name_size_field_width as usize; // Link name diff --git a/crates/clawhdf5-format/src/local_heap.rs b/crates/clawhdf5-format/src/local_heap.rs index 39e9b26..b2ef923 100644 --- a/crates/clawhdf5-format/src/local_heap.rs +++ b/crates/clawhdf5-format/src/local_heap.rs @@ -3,6 +3,7 @@ #[cfg(not(feature = "std"))] use alloc::string::String; +use crate::addr::to_usize; use crate::error::FormatError; /// Parsed HDF5 Local Heap header. @@ -140,15 +141,15 @@ impl LocalHeap { /// Read a null-terminated string from the heap's data segment at the given byte offset. pub fn read_string(&self, file_data: &[u8], string_offset: u64) -> Result { - let seg_addr = self.data_segment_address as usize; + let seg_addr = to_usize(self.data_segment_address)?; let str_start = seg_addr - .checked_add(string_offset as usize) + .checked_add(to_usize(string_offset)?) .ok_or(FormatError::Overflow( "local heap seg_addr + string_offset overflow".into(), ))?; let seg_end = seg_addr - .checked_add(self.data_segment_size as usize) + .checked_add(to_usize(self.data_segment_size)?) .ok_or(FormatError::Overflow( "local heap seg_addr + data_segment_size overflow".into(), ))?; diff --git a/crates/clawhdf5-format/src/object_header.rs b/crates/clawhdf5-format/src/object_header.rs index 1fc8696..3d7b7f7 100644 --- a/crates/clawhdf5-format/src/object_header.rs +++ b/crates/clawhdf5-format/src/object_header.rs @@ -5,6 +5,7 @@ use alloc::vec::Vec; use byteorder::{ByteOrder, LittleEndian}; +use crate::addr::to_usize; use crate::error::FormatError; use crate::message_type::MessageType; @@ -264,8 +265,8 @@ impl ObjectHeader { // Follow continuations (v1 continuation chunks are just raw // messages, no signature); check_message has checked the body. if msg_type == MessageType::ObjectHeaderContinuation { - let cont_offset = read_offset(body, 0, offset_size)? as usize; - let cont_length = read_offset(body, offset_size as usize, length_size)? as usize; + let cont_offset = to_usize(read_offset(body, 0, offset_size)?)?; + let cont_length = to_usize(read_offset(body, offset_size as usize, length_size)?)?; Self::parse_v1_chunk( data, cont_offset, @@ -339,7 +340,7 @@ impl ObjectHeader { _ => unreachable!(), }; ensure_len(data, pos, chunk_size_width as usize)?; - let chunk0_size = read_offset(data, pos, chunk_size_width)? as usize; + let chunk0_size = to_usize(read_offset(data, pos, chunk_size_width)?)?; pos += chunk_size_width as usize; // Bit 2: attribute creation order tracked → messages include creation order field let has_creation_order = flags & 0x04 != 0; @@ -472,8 +473,8 @@ impl ObjectHeader { let msg_type = MessageType::from_u16(msg_type_raw); if msg_type == MessageType::ObjectHeaderContinuation { // check_message has checked the body holds both fields. - let cont_off = read_offset(body, 0, offset_size)? as usize; - let cont_len = read_offset(body, offset_size as usize, length_size)? as usize; + let cont_off = to_usize(read_offset(body, 0, offset_size)?)?; + let cont_len = to_usize(read_offset(body, offset_size as usize, length_size)?)?; continuations.push((cont_off, cont_len)); } else if msg_type == MessageType::Nil { null_count += 1; diff --git a/crates/clawhdf5-format/src/partial_read.rs b/crates/clawhdf5-format/src/partial_read.rs index 4c5a01e..ad5dc8e 100644 --- a/crates/clawhdf5-format/src/partial_read.rs +++ b/crates/clawhdf5-format/src/partial_read.rs @@ -203,7 +203,12 @@ fn copy_overlap( }; let (src_strides, out_strides) = (strides(src_shape), strides(box_extent)); let last = rank - 1; - let run = ((hi[last] - lo[last]) as usize) * elem_size; + // Byte offsets into the in-memory buffers; one that does not fit `usize` + // (a 32-bit target) is out of both buffers, like one past their ends. + let bytes = |elements: u64| usize::try_from(elements).ok()?.checked_mul(elem_size); + let Some(run) = bytes(hi[last] - lo[last]) else { + return; + }; let mut idx = lo.clone(); loop { @@ -213,8 +218,12 @@ fn copy_overlap( let out_at: u64 = (0..rank) .map(|d| (idx[d] - box_start[d]) * out_strides[d]) .sum(); - let (s, o) = (src_at as usize * elem_size, out_at as usize * elem_size); - if let (Some(from), Some(to)) = (src.get(s..s + run), out.get_mut(o..o + run)) { + if let (Some(s), Some(o)) = (bytes(src_at), bytes(out_at)) + && let (Some(from), Some(to)) = ( + src.get(s..s.saturating_add(run)), + out.get_mut(o..o.saturating_add(run)), + ) + { to.copy_from_slice(from); } // Advance over every dimension but the last. diff --git a/crates/clawhdf5-format/src/selection.rs b/crates/clawhdf5-format/src/selection.rs index de31e46..3dbc6ee 100644 --- a/crates/clawhdf5-format/src/selection.rs +++ b/crates/clawhdf5-format/src/selection.rs @@ -19,6 +19,7 @@ use alloc::{vec, vec::Vec}; use core::ops::Range; +use crate::addr::to_usize; use crate::error::FormatError; /// A selection describing which elements of a dataset to access. @@ -562,7 +563,7 @@ fn decode_hyperslab(r: &mut SelReader, version: u64) -> Result 32 { @@ -625,11 +626,11 @@ fn decode_hyperslab(r: &mut SelReader, version: u64) -> Result { let target_header = - ObjectHeader::parse(file_data, addr as usize, offset_size, length_size)?; + ObjectHeader::parse(file_data, to_usize(addr)?, offset_size, length_size)?; for msg in &target_header.messages { if msg.msg_type == target_msg_type && !is_shared(msg.flags) { return Ok(msg.data.clone()); diff --git a/crates/clawhdf5-format/src/vds.rs b/crates/clawhdf5-format/src/vds.rs index d67b832..df1b068 100644 --- a/crates/clawhdf5-format/src/vds.rs +++ b/crates/clawhdf5-format/src/vds.rs @@ -15,6 +15,7 @@ #[cfg(not(feature = "std"))] use alloc::{format, string::String, vec, vec::Vec}; +use crate::addr::to_usize; use crate::data_layout::{DataLayout, VdsMapping, parse_vds_mappings}; use crate::dataspace::Dataspace; use crate::datatype::Datatype; @@ -208,7 +209,7 @@ fn load_mappings( return Ok(Vec::new()); }; let coll = - crate::global_heap::GlobalHeapCollection::parse(file_data, addr as usize, length_size)?; + crate::global_heap::GlobalHeapCollection::parse(file_data, to_usize(addr)?, length_size)?; let index = u16::try_from(*global_heap_index) .map_err(|_| vds_err("VDS mapping heap index out of range"))?; let obj = coll @@ -611,12 +612,12 @@ fn scatter( return Err(vds_err("virtual/source selection element counts differ")); } for (&v, &s) in vidx.iter().zip(sidx) { - let (vo, so) = (v as usize * elem_size, s as usize * elem_size); + let (vo, so) = (to_usize(v)? * elem_size, to_usize(s)? * elem_size); if vo + elem_size > out.len() || so + elem_size > src.len() { return Err(vds_err("virtual dataset selection out of bounds")); } out[vo..vo + elem_size].copy_from_slice(&src[so..so + elem_size]); - mapped[v as usize] = true; + mapped[to_usize(v)?] = true; } Ok(()) } @@ -747,7 +748,7 @@ fn selection_indices( return Err(vds_err("VDS selection blocks overlap")); } } - let mut out = Vec::with_capacity(volume as usize); + let mut out = Vec::with_capacity(to_usize(volume)?); for (s, e) in starts.chunks_exact(rank).zip(ends.chunks_exact(rank)) { let mut cur = s.to_vec(); 'block: loop { @@ -868,7 +869,7 @@ fn load_source_file(whole: &mut [u8]) -> Result<(), FormatError> { // read as before, up to its length. let end = sb .data_end(base as u64, whole.len() as u64) - .map_or(whole.len(), |e| base + e as usize); + .map_or(Ok(whole.len()), |e| to_usize(e).map(|e| base + e))?; crate::superblock_ext::apply_cache_image_in_place(&mut whole[base..end], &sb) } @@ -921,7 +922,7 @@ fn open_source(file_data: &[u8], path: &str) -> Result, Forma Err(FormatError::PathNotFound(_)) => return Ok(None), Err(e) => return Err(e), }; - let header = crate::object_header::ObjectHeader::parse(file_data, addr as usize, os, ls)?; + let header = crate::object_header::ObjectHeader::parse(file_data, to_usize(addr)?, os, ls)?; let mut src = OpenSource { offset_size: os, length_size: ls, diff --git a/crates/clawhdf5-format/src/vl_data.rs b/crates/clawhdf5-format/src/vl_data.rs index 3a54c1a..a998a62 100644 --- a/crates/clawhdf5-format/src/vl_data.rs +++ b/crates/clawhdf5-format/src/vl_data.rs @@ -9,6 +9,7 @@ use alloc::{collections::BTreeMap, format, string::String, vec, vec::Vec}; #[cfg(feature = "std")] use std::collections::BTreeMap; +use crate::addr::to_usize; use crate::error::FormatError; use crate::global_heap::{GlobalHeapCollection, GlobalHeapIndex}; @@ -55,7 +56,7 @@ pub fn parse_vl_references( ) -> Result, FormatError> { let elem_size = 4 + offset_size as usize + 4; // length + address + index let total = - (num_elements as usize) + to_usize(num_elements)? .checked_mul(elem_size) .ok_or(FormatError::UnexpectedEof { expected: usize::MAX, @@ -68,7 +69,7 @@ pub fn parse_vl_references( }); } - let mut elements = Vec::with_capacity(num_elements as usize); + let mut elements = Vec::with_capacity(to_usize(num_elements)?); let mut pos = 0; for _ in 0..num_elements { @@ -406,7 +407,7 @@ impl<'a> VlResolver<'a> { let index = GlobalHeapCollection::parse_index(self.file_data, offset, self.length_size)?; // parse_index checked that the collection lies in the file. - let end = offset + index.collection_size as usize; + let end = offset + to_usize(index.collection_size)?; self.check_overlap(offset, end)?; let coll = CachedCollection::new(index); if self.cached_bytes.saturating_add(coll.cost()) > self.budget { diff --git a/scripts/check-32bit-casts.sh b/scripts/check-32bit-casts.sh new file mode 100755 index 0000000..cfed6f5 --- /dev/null +++ b/scripts/check-32bit-casts.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# CI check: clawhdf5-format has no truncating `u64 as usize` cast on a 32-bit +# target. HDF5 addresses and lengths are 64-bit; on wasm32 (or any 32-bit +# target) such a cast silently wraps an address past 4 GiB onto another part +# of the file. File values go through `addr::to_usize` (a clean error) and +# in-memory counts through `addr::saturating_usize`. +# +# Lints wasm32-unknown-unknown with clippy's cast_possible_truncation and +# fails on any u64 -> usize finding (other truncations are not checked here). +# +# Usage: +# ./scripts/check-32bit-casts.sh +# +# Prerequisites: +# rustup target add wasm32-unknown-unknown + +set -euo pipefail + +TARGET="wasm32-unknown-unknown" +echo "==> Checking for truncating u64 -> usize casts in clawhdf5-format ($TARGET)" + +out=$(cargo clippy -p clawhdf5-format --target "$TARGET" \ + --features plugin-filters --message-format short \ + -- -A clippy::all -W clippy::cast_possible_truncation 2>&1) || { + echo "$out" + echo "==> clippy failed" >&2 + exit 1 +} +found=$(grep -F 'casting `u64` to `usize`' <<<"$out" || true) +if [ -n "$found" ]; then + echo "$found" + echo "==> use addr::to_usize (file values) or addr::saturating_usize (in-memory counts)" >&2 + exit 1 +fi +echo "==> no truncating u64 -> usize casts" diff --git a/scripts/ci-test.sh b/scripts/ci-test.sh index 3012cba..c7c77dd 100755 --- a/scripts/ci-test.sh +++ b/scripts/ci-test.sh @@ -147,6 +147,8 @@ run_step "wasm32 clippy (clawhdf5-wasm)" cargo clippy \ --target wasm32-unknown-unknown \ --all-targets \ -- -D warnings +# A 64-bit file address must not wrap on a 32-bit target. +run_step "check-32bit-casts.sh" "$SCRIPT_DIR/check-32bit-casts.sh" # The built wasm package, run under Node against h5py/netCDF4-written files, # and the viewer page in headless Chromium when one is found.