format: no truncating u64 -> usize casts

Every `u64 as usize` cast in clawhdf5-format (115 on wasm32) now goes
through addr::to_usize for values read from the file — addresses, lengths,
counts, dimensions: FormatError::Overflow where the value does not fit
instead of wrapping onto another part of the file on a 32-bit target — or
addr::saturating_usize for counts bounded by something in memory (codec
progress counters, writer sizes), which fail a bounds check or allocation
rather than wrap. A chunk whose offset does not fit lies outside the
dataset and is skipped; partial reads treat such an offset as out of the
buffers. On 64-bit targets nothing changes.

scripts/check-32bit-casts.sh (run by ci-test.sh) lints the wasm32 build
with clippy's cast_possible_truncation and fails on any u64 -> usize
finding; before this commit it listed 115.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-26 13:33:24 -05:00
co-authored by Claude Opus 5.5
parent 02e89c1d2d
commit b41583113a
27 changed files with 236 additions and 99 deletions
+22
View File
@@ -23,6 +23,19 @@ pub fn to_usize(value: u64) -> Result<usize, FormatError> {
usize::try_from(value).map_err(|_| too_large(value)) usize::try_from(value).map_err(|_| too_large(value))
} }
/// A count or offset into an in-memory buffer (a codec's progress counter,
/// a size the writer computed from data it holds) as a `usize`, saturating
/// at `usize::MAX` instead of truncating.
///
/// For values that are bounded by the length of something in memory, so
/// always fit; if one ever did not, a saturated index fails its bounds check
/// or allocation instead of silently addressing the wrong bytes. A value
/// read from the file uses [`to_usize`].
#[inline]
pub fn saturating_usize(value: u64) -> usize {
usize::try_from(value).unwrap_or(usize::MAX)
}
#[cold] #[cold]
#[inline(never)] #[inline(never)]
fn too_large(value: u64) -> FormatError { fn too_large(value: u64) -> FormatError {
@@ -42,6 +55,15 @@ mod tests {
assert_eq!(to_usize(usize::MAX as u64), Ok(usize::MAX)); assert_eq!(to_usize(usize::MAX as u64), Ok(usize::MAX));
} }
#[test]
fn saturating_conversion_never_wraps() {
assert_eq!(saturating_usize(0), 0);
assert_eq!(saturating_usize(0x1234), 0x1234);
assert_eq!(saturating_usize(usize::MAX as u64), usize::MAX);
// Past usize::MAX (32-bit targets) or at u64::MAX: saturates.
assert_eq!(saturating_usize(u64::MAX), usize::MAX);
}
#[test] #[test]
fn values_past_usize_max_are_an_error_not_truncated() { fn values_past_usize_max_are_an_error_not_truncated() {
// Only reachable where usize is narrower than u64; on a 64-bit host // Only reachable where usize is narrower than u64; on a 64-bit host
+7 -1
View File
@@ -3,6 +3,7 @@
#[cfg(not(feature = "std"))] #[cfg(not(feature = "std"))]
use alloc::vec::Vec; use alloc::vec::Vec;
use crate::addr::to_usize;
use crate::error::FormatError; use crate::error::FormatError;
/// A parsed B-tree v1 node. /// A parsed B-tree v1 node.
@@ -164,7 +165,12 @@ fn collect_symbol_table_nodes_inner(
return Err(FormatError::NestingDepthExceeded); return Err(FormatError::NestingDepthExceeded);
} }
let node = BTreeV1Node::parse(file_data, btree_address as usize, offset_size, length_size)?; let node = BTreeV1Node::parse(
file_data,
to_usize(btree_address)?,
offset_size,
length_size,
)?;
if node.node_type != 0 { if node.node_type != 0 {
return Err(FormatError::InvalidBTreeNodeType(node.node_type)); return Err(FormatError::InvalidBTreeNodeType(node.node_type));
+5 -2
View File
@@ -6,6 +6,7 @@
//! libhdf5 uses (`H5B2__hdr_init`) and the reader decodes pointers with, so //! libhdf5 uses (`H5B2__hdr_init`) and the reader decodes pointers with, so
//! the pointer widths the writer encodes are the ones every reader expects. //! the pointer widths the writer encodes are the ones every reader expects.
use crate::addr::saturating_usize;
#[cfg(not(feature = "std"))] #[cfg(not(feature = "std"))]
use alloc::{format, vec, vec::Vec}; use alloc::{format, vec, vec::Vec};
@@ -105,7 +106,9 @@ pub(crate) fn build_btree_v2(
first_node: addr + hdr_len as u64, first_node: addr + hdr_len as u64,
nodes: Vec::new(), nodes: Vec::new(),
}; };
let root = (n > 0).then(|| w.node(depth, 0, n as usize)).transpose()?; let root = (n > 0)
.then(|| w.node(depth, 0, saturating_usize(n)))
.transpose()?;
let mut out = Vec::with_capacity(hdr_len + w.nodes.len() * p.node_size as usize); let mut out = Vec::with_capacity(hdr_len + w.nodes.len() * p.node_size as usize);
out.extend_from_slice(b"BTHD"); out.extend_from_slice(b"BTHD");
@@ -201,7 +204,7 @@ impl TreeWriter<'_> {
"cannot spread {n} B-tree v2 records over {k} children at depth {depth}" "cannot spread {n} B-tree v2 records over {k} children at depth {depth}"
))); )));
} }
let k = k as usize; let k = saturating_usize(k);
let in_children = n - (k - 1); let in_children = n - (k - 1);
let (base, extra) = (in_children / k, in_children % k); let (base, extra) = (in_children / k, in_children % k);
+10 -1
View File
@@ -18,6 +18,7 @@ use alloc::collections::BTreeMap;
#[cfg(feature = "std")] #[cfg(feature = "std")]
use std::collections::HashMap; use std::collections::HashMap;
use crate::addr::to_usize;
use crate::chunk_cache::ChunkCoord; use crate::chunk_cache::ChunkCoord;
use crate::chunked_read::ChunkInfo; use crate::chunked_read::ChunkInfo;
@@ -167,7 +168,15 @@ impl ChunkLayout {
for (_coord, ci) in index.iter() { for (_coord, ci) in index.iter() {
let coord: ChunkCoord = ci.offsets.iter().take(rank).copied().collect(); let coord: ChunkCoord = ci.offsets.iter().take(rank).copied().collect();
let chunk_offsets: Vec<usize> = coord.iter().map(|&o| o as usize).collect(); // `ds_dims` are `usize`: a chunk at an offset past `usize::MAX`
// (only on a 32-bit target) lies outside the dataset.
let Ok(chunk_offsets) = coord
.iter()
.map(|&o| to_usize(o))
.collect::<Result<Vec<usize>, _>>()
else {
continue;
};
let copies = if rank == 0 { let copies = if rank == 0 {
// Scalar dataset — single copy // Scalar dataset — single copy
+32 -14
View File
@@ -6,6 +6,7 @@ extern crate alloc;
#[cfg(not(feature = "std"))] #[cfg(not(feature = "std"))]
use alloc::{format, vec, vec::Vec}; use alloc::{format, vec, vec::Vec};
use crate::addr::to_usize;
#[cfg(feature = "std")] #[cfg(feature = "std")]
use crate::chunk_cache::{CacheAlignedBuffer, ChunkCache}; use crate::chunk_cache::{CacheAlignedBuffer, ChunkCache};
use crate::data_layout::DataLayout; use crate::data_layout::DataLayout;
@@ -265,7 +266,7 @@ fn fill_from_chunks(
))); )));
} }
let offsets = &c.offsets[..rank]; let offsets = &c.offsets[..rank];
let c_addr = c.address as usize; let c_addr = to_usize(c.address)?;
let size = c.chunk_size as usize; let size = c.chunk_size as usize;
ensure_len(file_data, c_addr, size)?; ensure_len(file_data, c_addr, size)?;
let raw = &file_data[c_addr..c_addr + size]; let raw = &file_data[c_addr..c_addr + size];
@@ -825,7 +826,7 @@ fn parse_chunk_node(
return Err(FormatError::NestingDepthExceeded); return Err(FormatError::NestingDepthExceeded);
} }
let offset = btree_address as usize; let offset = to_usize(btree_address)?;
let os = offset_size as usize; let os = offset_size as usize;
// Parse B-tree v1 header // Parse B-tree v1 header
@@ -945,7 +946,8 @@ pub fn generate_implicit_chunks(
} }
let total_chunks: u64 = num_chunks_per_dim.iter().product(); let total_chunks: u64 = num_chunks_per_dim.iter().product();
let mut chunks = Vec::with_capacity(total_chunks as usize); // A capacity hint only (a count past `usize::MAX` could not be pushed).
let mut chunks = Vec::with_capacity(usize::try_from(total_chunks).unwrap_or(0));
for linear_idx in 0..total_chunks { for linear_idx in 0..total_chunks {
let mut offsets = vec![0u64; rank]; let mut offsets = vec![0u64; rank];
let mut remaining = linear_idx; let mut remaining = linear_idx;
@@ -993,7 +995,7 @@ fn read_btree_v2_chunks(
use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records}; use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records};
let bad = |what: &str| FormatError::ChunkedReadError(format!("B-tree v2 chunk index: {what}")); let bad = |what: &str| FormatError::ChunkedReadError(format!("B-tree v2 chunk index: {what}"));
let header = BTreeV2Header::parse(file_data, addr as usize, offset_size, length_size)?; let header = BTreeV2Header::parse(file_data, to_usize(addr)?, offset_size, length_size)?;
let rank = chunk_dims.len(); let rank = chunk_dims.len();
let os = offset_size as usize; let os = offset_size as usize;
let record_size = header.record_size as usize; let record_size = header.record_size as usize;
@@ -1122,7 +1124,11 @@ pub fn list_chunks(
// Both v3 and v4 include element size as last dim (rank+1) // Both v3 and v4 include element size as last dim (rank+1)
let (rank, chunk_dims) = chunk_geometry(chunk_dimensions, version, dataspace, elem_size)?; let (rank, chunk_dims) = chunk_geometry(chunk_dimensions, version, dataspace, elem_size)?;
let ds_dims: Vec<usize> = dataspace.dimensions.iter().map(|&d| d as usize).collect(); let ds_dims: Vec<usize> = dataspace
.dimensions
.iter()
.map(|&d| to_usize(d))
.collect::<Result<_, _>>()?;
// Collect chunks based on version and index type // Collect chunks based on version and index type
let mut chunks = match (version, chunk_index_type) { let mut chunks = match (version, chunk_index_type) {
@@ -1158,7 +1164,7 @@ pub fn list_chunks(
// Fixed Array — use spatial chunk dims only // Fixed Array — use spatial chunk dims only
let spatial_chunk_dims: &[u32] = &chunk_dimensions[..rank]; let spatial_chunk_dims: &[u32] = &chunk_dimensions[..rank];
let header = let header =
FixedArrayHeader::parse(file_data, addr as usize, offset_size, length_size)?; FixedArrayHeader::parse(file_data, to_usize(addr)?, offset_size, length_size)?;
read_fixed_array_chunks( read_fixed_array_chunks(
file_data, file_data,
&header, &header,
@@ -1174,7 +1180,7 @@ pub fn list_chunks(
// Extensible Array — use spatial chunk dims only // Extensible Array — use spatial chunk dims only
let spatial_chunk_dims: &[u32] = &chunk_dimensions[..rank]; let spatial_chunk_dims: &[u32] = &chunk_dimensions[..rank];
let header = let header =
ExtensibleArrayHeader::parse(file_data, addr as usize, offset_size, length_size)?; ExtensibleArrayHeader::parse(file_data, to_usize(addr)?, offset_size, length_size)?;
read_extensible_array_chunks( read_extensible_array_chunks(
file_data, file_data,
&header, &header,
@@ -1349,7 +1355,11 @@ pub(crate) fn read_chunked_full<O>(
// dimension the total is 0 even if other dimensions are huge. // dimension the total is 0 even if other dimensions are huge.
return Ok(output); return Ok(output);
} }
let ds_dims: Vec<usize> = dataspace.dimensions.iter().map(|&d| d as usize).collect(); let ds_dims: Vec<usize> = dataspace
.dimensions
.iter()
.map(|&d| to_usize(d))
.collect::<Result<_, _>>()?;
let placer = ChunkPlacer::new(&chunk_dims, &ds_dims, elem_size); let placer = ChunkPlacer::new(&chunk_dims, &ds_dims, elem_size);
let chunk_total_bytes = checked_chunk_byte_len(&chunk_dims, elem_size)?; let chunk_total_bytes = checked_chunk_byte_len(&chunk_dims, elem_size)?;
// Chunks are cached only when the whole dataset fits: pushing a larger // Chunks are cached only when the whole dataset fits: pushing a larger
@@ -1603,7 +1613,11 @@ pub fn read_chunked_data_sweep(
check_chunk_element_size(layout, datatype, offset_size)?; check_chunk_element_size(layout, datatype, offset_size)?;
let elem_size = datatype.type_size() as usize; let elem_size = datatype.type_size() as usize;
let (rank, chunk_dims) = chunk_geometry(chunk_dimensions, version, dataspace, elem_size)?; let (rank, chunk_dims) = chunk_geometry(chunk_dimensions, version, dataspace, elem_size)?;
let ds_dims: Vec<usize> = dataspace.dimensions.iter().map(|&d| d as usize).collect(); let ds_dims: Vec<usize> = dataspace
.dimensions
.iter()
.map(|&d| to_usize(d))
.collect::<Result<_, _>>()?;
// The per-file cache is shared across datasets (and threads); every // The per-file cache is shared across datasets (and threads); every
// lookup is keyed by this dataset's chunk-index address, so another // lookup is keyed by this dataset's chunk-index address, so another
@@ -1659,7 +1673,7 @@ pub fn read_chunked_data_sweep(
cached cached
} else { } else {
// Decompress from file // Decompress from file
let c_addr = chunk_info.address as usize; let c_addr = to_usize(chunk_info.address)?;
let size = chunk_info.chunk_size as usize; let size = chunk_info.chunk_size as usize;
ensure_len(file_data, c_addr, size)?; ensure_len(file_data, c_addr, size)?;
let raw_chunk = &file_data[c_addr..c_addr + size]; let raw_chunk = &file_data[c_addr..c_addr + size];
@@ -1682,8 +1696,8 @@ pub fn read_chunked_data_sweep(
.offsets .offsets
.iter() .iter()
.take(rank) .take(rank)
.map(|&o| o as usize) .map(|&o| to_usize(o))
.collect(); .collect::<Result<_, _>>()?;
if rank == 0 { if rank == 0 {
let copy_len = decompressed.len().min(output.len()); let copy_len = decompressed.len().min(output.len());
@@ -1743,7 +1757,11 @@ pub fn read_chunked_data_indexed(
check_chunk_element_size(layout, datatype, offset_size)?; check_chunk_element_size(layout, datatype, offset_size)?;
let elem_size = datatype.type_size() as usize; let elem_size = datatype.type_size() as usize;
let (rank, chunk_dims) = chunk_geometry(chunk_dimensions, version, dataspace, elem_size)?; let (rank, chunk_dims) = chunk_geometry(chunk_dimensions, version, dataspace, elem_size)?;
let ds_dims: Vec<usize> = dataspace.dimensions.iter().map(|&d| d as usize).collect(); let ds_dims: Vec<usize> = dataspace
.dimensions
.iter()
.map(|&d| to_usize(d))
.collect::<Result<_, _>>()?;
// Chunk index and assembly plan for this dataset, built on first access // Chunk index and assembly plan for this dataset, built on first access
// and kept per dataset (keyed by chunk-index address) in the shared cache. // and kept per dataset (keyed by chunk-index address) in the shared cache.
@@ -1776,7 +1794,7 @@ pub fn read_chunked_data_indexed(
if let Some(cached) = cache.get_decompressed_in(addr, coord) { if let Some(cached) = cache.get_decompressed_in(addr, coord) {
chunk_buffers.push(cached); chunk_buffers.push(cached);
} else { } else {
let c_addr = *file_offset as usize; let c_addr = to_usize(*file_offset)?;
let size = *file_size as usize; let size = *file_size as usize;
ensure_len(file_data, c_addr, size)?; ensure_len(file_data, c_addr, size)?;
let raw_chunk = &file_data[c_addr..c_addr + size]; let raw_chunk = &file_data[c_addr..c_addr + size];
+12 -11
View File
@@ -3,6 +3,7 @@
#[cfg(not(feature = "std"))] #[cfg(not(feature = "std"))]
extern crate alloc; extern crate alloc;
use crate::addr::saturating_usize;
#[cfg(not(feature = "std"))] #[cfg(not(feature = "std"))]
use alloc::{format, vec, vec::Vec}; use alloc::{format, vec, vec::Vec};
@@ -414,18 +415,18 @@ pub fn split_into_chunks(
// Dataset strides (row-major) // Dataset strides (row-major)
let mut ds_strides = vec![1usize; rank]; let mut ds_strides = vec![1usize; rank];
for i in (0..rank.saturating_sub(1)).rev() { for i in (0..rank.saturating_sub(1)).rev() {
ds_strides[i] = ds_strides[i + 1] * shape[i + 1] as usize; ds_strides[i] = ds_strides[i + 1] * saturating_usize(shape[i + 1]);
} }
// Chunk strides // Chunk strides
let mut chunk_strides = vec![1usize; rank]; let mut chunk_strides = vec![1usize; rank];
for i in (0..rank.saturating_sub(1)).rev() { for i in (0..rank.saturating_sub(1)).rev() {
chunk_strides[i] = chunk_strides[i + 1] * chunk_dims[i + 1] as usize; chunk_strides[i] = chunk_strides[i + 1] * saturating_usize(chunk_dims[i + 1]);
} }
let chunk_total_elements: usize = chunk_dims.iter().map(|&d| d as usize).product(); let chunk_total_elements: usize = chunk_dims.iter().map(|&d| saturating_usize(d)).product();
let mut result = Vec::with_capacity(total_chunks as usize); let mut result = Vec::with_capacity(saturating_usize(total_chunks));
for linear_idx in 0..total_chunks { for linear_idx in 0..total_chunks {
// Convert linear index to chunk grid coordinates // Convert linear index to chunk grid coordinates
@@ -453,8 +454,8 @@ pub fn split_into_chunks(
let coord_in_chunk = remaining_idx / chunk_strides[d]; let coord_in_chunk = remaining_idx / chunk_strides[d];
remaining_idx %= chunk_strides[d]; remaining_idx %= chunk_strides[d];
let global_coord = offsets[d] as usize + coord_in_chunk; let global_coord = saturating_usize(offsets[d]) + coord_in_chunk;
if global_coord >= shape[d] as usize { if global_coord >= saturating_usize(shape[d]) {
out_of_bounds = true; out_of_bounds = true;
break; break;
} }
@@ -1036,7 +1037,7 @@ impl ChunkIndexPlan {
Ok(Self::SingleChunk) Ok(Self::SingleChunk)
} else { } else {
let grid = ChunkGrid::fixed_array(shape, Some(max), chunk_dims)?; let grid = ChunkGrid::fixed_array(shape, Some(max), chunk_dims)?;
Ok(Self::FixedArray(grid, nslots as usize)) Ok(Self::FixedArray(grid, saturating_usize(nslots)))
} }
} }
1 => Ok(Self::ExtensibleArray(ChunkGrid::extensible_array( 1 => Ok(Self::ExtensibleArray(ChunkGrid::extensible_array(
@@ -1251,7 +1252,7 @@ pub fn write_selection_to_buffer(
let rank = dims.len(); let rank = dims.len();
let mut ds_strides = vec![1usize; rank]; let mut ds_strides = vec![1usize; rank];
for i in (0..rank.saturating_sub(1)).rev() { for i in (0..rank.saturating_sub(1)).rev() {
ds_strides[i] = ds_strides[i + 1] * dims[i + 1] as usize; ds_strides[i] = ds_strides[i + 1] * saturating_usize(dims[i + 1]);
} }
let mut src_offset = 0usize; let mut src_offset = 0usize;
@@ -1301,7 +1302,7 @@ pub fn write_selection_to_buffer(
buffer, buffer,
new_data, new_data,
src_offset, src_offset,
current_ds_offset + coord as usize * ds_strides[d], current_ds_offset + saturating_usize(coord) * ds_strides[d],
); );
} }
} }
@@ -1328,14 +1329,14 @@ pub fn write_selection_to_buffer(
let rank = dims.len(); let rank = dims.len();
let mut ds_strides = vec![1usize; rank]; let mut ds_strides = vec![1usize; rank];
for i in (0..rank.saturating_sub(1)).rev() { for i in (0..rank.saturating_sub(1)).rev() {
ds_strides[i] = ds_strides[i + 1] * dims[i + 1] as usize; ds_strides[i] = ds_strides[i + 1] * saturating_usize(dims[i + 1]);
} }
for (pi, pt) in pts.iter().enumerate() { for (pi, pt) in pts.iter().enumerate() {
let flat: usize = pt let flat: usize = pt
.iter() .iter()
.zip(ds_strides.iter()) .zip(ds_strides.iter())
.map(|(&p, &s)| p as usize * s) .map(|(&p, &s)| saturating_usize(p) * s)
.sum(); .sum();
let dst = flat * elem_size; let dst = flat * elem_size;
let src = pi * elem_size; let src = pi * elem_size;
+3 -2
View File
@@ -6,6 +6,7 @@ use alloc::{format, string::String, vec::Vec};
#[cfg(feature = "std")] #[cfg(feature = "std")]
use std::string::String; use std::string::String;
use crate::addr::to_usize;
use crate::error::FormatError; use crate::error::FormatError;
/// A single VDS (Virtual Dataset) source mapping. /// A single VDS (Virtual Dataset) source mapping.
@@ -207,7 +208,7 @@ pub fn parse_vds_mappings(
"VDS mapping shares a name with a later entry".into(), "VDS mapping shares a name with a later entry".into(),
)); ));
} }
Ok(idx as usize) to_usize(idx)
}; };
let source_file = if flags & VDS_SOURCE_SAME_FILE != 0 { let source_file = if flags & VDS_SOURCE_SAME_FILE != 0 {
@@ -320,7 +321,7 @@ impl DataLayout {
{ {
let coll = crate::global_heap::GlobalHeapCollection::parse( let coll = crate::global_heap::GlobalHeapCollection::parse(
file_data, file_data,
addr as usize, to_usize(addr)?,
length_size, length_size,
)?; )?;
let obj = coll.get_object(*global_heap_index as u16).ok_or( let obj = coll.get_object(*global_heap_index as u16).ok_or(
+9 -8
View File
@@ -6,6 +6,7 @@ use alloc::{collections::BTreeMap, format, string::String, vec, vec::Vec};
#[cfg(feature = "std")] #[cfg(feature = "std")]
use std::collections::BTreeMap; use std::collections::BTreeMap;
use crate::addr::to_usize;
#[cfg(feature = "std")] #[cfg(feature = "std")]
use crate::chunk_cache::ChunkCache; use crate::chunk_cache::ChunkCache;
use crate::chunked_read::read_chunked_data; use crate::chunked_read::read_chunked_data;
@@ -117,7 +118,7 @@ pub fn read_raw_data_zerocopy<'a>(
dataspace: &Dataspace, dataspace: &Dataspace,
datatype: &Datatype, datatype: &Datatype,
) -> Result<Option<&'a [u8]>, FormatError> { ) -> Result<Option<&'a [u8]>, FormatError> {
let num_elements = dataspace.num_elements() as usize; let num_elements = to_usize(dataspace.num_elements())?;
let elem_size = datatype.type_size() as usize; let elem_size = datatype.type_size() as usize;
let expected_size = num_elements.checked_mul(elem_size).ok_or_else(|| { let expected_size = num_elements.checked_mul(elem_size).ok_or_else(|| {
FormatError::Overflow(format!( FormatError::Overflow(format!(
@@ -128,7 +129,7 @@ pub fn read_raw_data_zerocopy<'a>(
match layout { match layout {
DataLayout::Contiguous { address, size } => { DataLayout::Contiguous { address, size } => {
let addr = address.ok_or(FormatError::NoDataAllocated)?; let addr = address.ok_or(FormatError::NoDataAllocated)?;
let addr = addr as usize; let addr = to_usize(addr)?;
let sz = contiguous_read_len(*size, expected_size)?; let sz = contiguous_read_len(*size, expected_size)?;
ensure_len(file_data, addr, sz)?; ensure_len(file_data, addr, sz)?;
Ok(Some(&file_data[addr..addr + sz])) Ok(Some(&file_data[addr..addr + sz]))
@@ -219,7 +220,7 @@ fn read_raw_data_full_impl(
length_size: u8, length_size: u8,
resolver: Option<&VdsSourceResolver>, resolver: Option<&VdsSourceResolver>,
) -> Result<Vec<u8>, FormatError> { ) -> Result<Vec<u8>, FormatError> {
let num_elements = dataspace.num_elements() as usize; let num_elements = to_usize(dataspace.num_elements())?;
let elem_size = datatype.type_size() as usize; let elem_size = datatype.type_size() as usize;
let expected_size = num_elements.checked_mul(elem_size).ok_or_else(|| { let expected_size = num_elements.checked_mul(elem_size).ok_or_else(|| {
FormatError::Overflow(format!( FormatError::Overflow(format!(
@@ -239,7 +240,7 @@ fn read_raw_data_full_impl(
} }
DataLayout::Contiguous { address, size } => { DataLayout::Contiguous { address, size } => {
let addr = address.ok_or(FormatError::NoDataAllocated)?; let addr = address.ok_or(FormatError::NoDataAllocated)?;
let addr = addr as usize; let addr = to_usize(addr)?;
let sz = contiguous_read_len(*size, expected_size)?; let sz = contiguous_read_len(*size, expected_size)?;
ensure_len(file_data, addr, sz)?; ensure_len(file_data, addr, sz)?;
let mut out = crate::bulk_alloc::vec_for_bulk(sz); let mut out = crate::bulk_alloc::vec_for_bulk(sz);
@@ -582,7 +583,7 @@ pub fn extract_selection_from_buffer(
let rank = dims.len(); let rank = dims.len();
let mut ds_strides = vec![1usize; rank]; let mut ds_strides = vec![1usize; rank];
for i in (0..rank.saturating_sub(1)).rev() { for i in (0..rank.saturating_sub(1)).rev() {
ds_strides[i] = ds_strides[i + 1] * dims[i + 1] as usize; ds_strides[i] = ds_strides[i + 1] * to_usize(dims[i + 1])?;
} }
let mut output = Vec::with_capacity(pts.len() * elem_size); let mut output = Vec::with_capacity(pts.len() * elem_size);
@@ -590,8 +591,8 @@ pub fn extract_selection_from_buffer(
let flat: usize = pt let flat: usize = pt
.iter() .iter()
.zip(ds_strides.iter()) .zip(ds_strides.iter())
.map(|(&p, &s)| p as usize * s) .map(|(&p, &s)| Ok(to_usize(p)? * s))
.sum(); .sum::<Result<usize, FormatError>>()?;
let src = flat * elem_size; let src = flat * elem_size;
if src + elem_size <= full_data.len() { if src + elem_size <= full_data.len() {
output.extend_from_slice(&full_data[src..src + elem_size]); output.extend_from_slice(&full_data[src..src + elem_size]);
@@ -1341,7 +1342,7 @@ pub fn read_compound_fields(
let mut fields = Vec::with_capacity(members.len()); let mut fields = Vec::with_capacity(members.len());
for m in members { for m in members {
let field_size = m.datatype.type_size() as usize; let field_size = m.datatype.type_size() as usize;
let offset = m.byte_offset as usize; let offset = to_usize(m.byte_offset)?;
if offset if offset
.checked_add(field_size) .checked_add(field_size)
.is_none_or(|end| end > elem_size) .is_none_or(|end| end > elem_size)
+2 -1
View File
@@ -3,6 +3,7 @@
#[cfg(not(feature = "std"))] #[cfg(not(feature = "std"))]
extern crate alloc; extern crate alloc;
use crate::addr::saturating_usize;
#[cfg(not(feature = "std"))] #[cfg(not(feature = "std"))]
use alloc::{vec, vec::Vec}; use alloc::{vec, vec::Vec};
@@ -247,7 +248,7 @@ pub fn build_extensible_array_at(
// Header (EAHD). The six statistics are, in order: super blocks, their // Header (EAHD). The six statistics are, in order: super blocks, their
// bytes, data blocks, their bytes, max index set, elements realised. // bytes, data blocks, their bytes, max index set, elements realised.
let mut out = Vec::with_capacity((cursor - ea_base_address) as usize); let mut out = Vec::with_capacity(saturating_usize(cursor - ea_base_address));
out.extend_from_slice(b"EAHD"); out.extend_from_slice(b"EAHD");
out.push(0); // version out.push(0); // version
out.push(client_id); out.push(client_id);
@@ -9,6 +9,7 @@ extern crate alloc;
#[cfg(not(feature = "std"))] #[cfg(not(feature = "std"))]
use alloc::{format, vec, vec::Vec}; use alloc::{format, vec, vec::Vec};
use crate::addr::to_usize;
use crate::chunk_grid::ChunkGrid; use crate::chunk_grid::ChunkGrid;
use crate::chunked_read::ChunkInfo; use crate::chunked_read::ChunkInfo;
use crate::error::FormatError; use crate::error::FormatError;
@@ -451,7 +452,7 @@ pub fn read_extensible_array_chunks(
// Parse index block (EAIB): signature(4) + version(1) + client_id(1) // Parse index block (EAIB): signature(4) + version(1) + client_id(1)
// + header address(offset_size), then the inline elements, then the // + header address(offset_size), then the inline elements, then the
// direct data block addresses, then the super block addresses. // direct data block addresses, then the super block addresses.
let ib_offset = header.index_block_address as usize; let ib_offset = to_usize(header.index_block_address)?;
let ib_header_size = 4 + 1 + 1 + os; let ib_header_size = 4 + 1 + 1 + os;
ensure_len(file_data, ib_offset, ib_header_size)?; ensure_len(file_data, ib_offset, ib_header_size)?;
@@ -463,7 +464,7 @@ pub fn read_extensible_array_chunks(
let mut pos = ib_offset + ib_header_size; let mut pos = ib_offset + ib_header_size;
let mut chunks = Vec::new(); let mut chunks = Vec::new();
let total_elements = header.num_elements as usize; let total_elements = to_usize(header.num_elements)?;
let dmin = header.min_dblk_nelmts as usize; let dmin = header.min_dblk_nelmts as usize;
if dmin == 0 || !dmin.is_power_of_two() { if dmin == 0 || !dmin.is_power_of_two() {
@@ -563,7 +564,7 @@ pub fn read_extensible_array_chunks(
} }
chunks.extend(read_data_block_elements( chunks.extend(read_data_block_elements(
file_data, file_data,
addr as usize, to_usize(addr)?,
dblk_nelmts, dblk_nelmts,
header, header,
offset_size, offset_size,
@@ -592,7 +593,7 @@ pub fn read_extensible_array_chunks(
if !is_undefined_addr(sb_addr, offset_size) { if !is_undefined_addr(sb_addr, offset_size) {
chunks.extend(read_super_block( chunks.extend(read_super_block(
file_data, file_data,
sb_addr as usize, to_usize(sb_addr)?,
ndblks, ndblks,
dblk_nelmts, dblk_nelmts,
header, header,
@@ -676,7 +677,7 @@ fn read_super_block(
if !is_undefined_addr(addr, offset_size) { if !is_undefined_addr(addr, offset_size) {
chunks.extend(read_data_block_elements( chunks.extend(read_data_block_elements(
file_data, file_data,
addr as usize, to_usize(addr)?,
dblk_nelmts, dblk_nelmts,
header, header,
offset_size, offset_size,
+11 -9
View File
@@ -3,6 +3,7 @@
//! Produces valid HDF5 files with v3 superblock, v2 object headers, //! Produces valid HDF5 files with v3 superblock, v2 object headers,
//! link messages, contiguous datasets, inline and dense attributes. //! link messages, contiguous datasets, inline and dense attributes.
use crate::addr::saturating_usize;
#[cfg(not(feature = "std"))] #[cfg(not(feature = "std"))]
use alloc::{format, vec, vec::Vec}; use alloc::{format, vec, vec::Vec};
@@ -336,7 +337,7 @@ pub(crate) fn build_single_block_fractal_heap(
// An object must fit one direct block: the writer has no huge-object // An object must fit one direct block: the writer has no huge-object
// path, and libhdf5 cannot read an object that overruns its block. // path, and libhdf5 cannot read an object that overruns its block.
let max_managed = max_direct_block_size as usize - dblock_header_size; let max_managed = saturating_usize(max_direct_block_size) - dblock_header_size;
if let Some(big) = serialized.iter().find(|s| s.len() > max_managed) { if let Some(big) = serialized.iter().find(|s| s.len() > max_managed) {
return Err(FormatError::SerializationError(format!( return Err(FormatError::SerializationError(format!(
"a {}-byte message cannot go in dense storage: a fractal heap \ "a {}-byte message cannot go in dense storage: a fractal heap \
@@ -392,7 +393,7 @@ pub(crate) fn build_single_block_fractal_heap(
let dblock_addr = frhp_addr + frhp_size as u64; let dblock_addr = frhp_addr + frhp_size as u64;
let btree_addr = dblock_addr + starting_block_size; let btree_addr = dblock_addr + starting_block_size;
let data_space = starting_block_size as usize - dblock_header_size; let data_space = saturating_usize(starting_block_size) - dblock_header_size;
let free_space = data_space - total_data_size; let free_space = data_space - total_data_size;
// Build fractal heap header // Build fractal heap header
@@ -428,7 +429,7 @@ pub(crate) fn build_single_block_fractal_heap(
debug_assert_eq!(frhp.len(), frhp_size); debug_assert_eq!(frhp.len(), frhp_size);
// Build direct block: header (with checksum) + data + padding // Build direct block: header (with checksum) + data + padding
let mut dblock = Vec::with_capacity(starting_block_size as usize); let mut dblock = Vec::with_capacity(saturating_usize(starting_block_size));
dblock.extend_from_slice(b"FHDB"); dblock.extend_from_slice(b"FHDB");
dblock.push(0); // version dblock.push(0); // version
write_offset(&mut dblock, frhp_addr, OFFSET_SIZE); write_offset(&mut dblock, frhp_addr, OFFSET_SIZE);
@@ -446,12 +447,12 @@ pub(crate) fn build_single_block_fractal_heap(
} }
// Pad to full block size // Pad to full block size
dblock.resize(starting_block_size as usize, 0); dblock.resize(saturating_usize(starting_block_size), 0);
// Checksum: computed over entire block with checksum field zeroed // Checksum: computed over entire block with checksum field zeroed
let dblock_checksum = crate::checksum::jenkins_lookup3(&dblock); let dblock_checksum = crate::checksum::jenkins_lookup3(&dblock);
dblock[cksum_pos..cksum_pos + 4].copy_from_slice(&dblock_checksum.to_le_bytes()); dblock[cksum_pos..cksum_pos + 4].copy_from_slice(&dblock_checksum.to_le_bytes());
debug_assert_eq!(dblock.len(), starting_block_size as usize); debug_assert_eq!(dblock.len(), saturating_usize(starting_block_size));
// Build heap IDs // Build heap IDs
let heap_ids: Vec<Vec<u8>> = obj_offsets let heap_ids: Vec<Vec<u8>> = obj_offsets
@@ -706,7 +707,7 @@ impl HeapIndirectBlock {
let cksum_pos = out.len(); let cksum_pos = out.len();
out.extend_from_slice(&[0u8; 4]); // checksum placeholder out.extend_from_slice(&[0u8; 4]); // checksum placeholder
out.extend_from_slice(&b.data); out.extend_from_slice(&b.data);
out.resize(d + b.size as usize, 0); out.resize(d + saturating_usize(b.size), 0);
let cksum = crate::checksum::jenkins_lookup3(&out[d..]); let cksum = crate::checksum::jenkins_lookup3(&out[d..]);
out[cksum_pos..cksum_pos + 4].copy_from_slice(&cksum.to_le_bytes()); out[cksum_pos..cksum_pos + 4].copy_from_slice(&cksum.to_le_bytes());
child += b.size; child += b.size;
@@ -740,7 +741,7 @@ impl HeapPacker<'_> {
nrows: Option<usize>, nrows: Option<usize>,
) -> Result<HeapIndirectBlock, FormatError> { ) -> Result<HeapIndirectBlock, FormatError> {
let geom = self.geom; let geom = self.geom;
let width = geom.width as usize; let width = saturating_usize(geom.width);
let mut slots = Vec::new(); let mut slots = Vec::new();
let mut off = heap_offset; let mut off = heap_offset;
let mut row = 0usize; let mut row = 0usize;
@@ -763,7 +764,8 @@ impl HeapPacker<'_> {
// A child whose biggest direct block cannot hold the // A child whose biggest direct block cannot hold the
// next object is skipped whole, not walked. // next object is skipped whole, not walked.
let biggest = geom.row_size(child_rows.min(geom.max_direct_rows()) - 1); let biggest = geom.row_size(child_rows.min(geom.max_direct_rows()) - 1);
if self.objects[self.next].len() > (biggest as usize - geom.dblock_header_size) if self.objects[self.next].len()
> (saturating_usize(biggest) - geom.dblock_header_size)
{ {
slots.push(HeapSlot::Empty); slots.push(HeapSlot::Empty);
off += size; off += size;
@@ -794,7 +796,7 @@ impl HeapPacker<'_> {
/// objects as fit; leave it unallocated if not even the next one does. /// objects as fit; leave it unallocated if not even the next one does.
fn fill_direct(&mut self, heap_offset: u64, size: u64) -> HeapSlot { fn fill_direct(&mut self, heap_offset: u64, size: u64) -> HeapSlot {
let header = self.geom.dblock_header_size; let header = self.geom.dblock_header_size;
let capacity = size as usize - header; let capacity = saturating_usize(size) - header;
let mut data = Vec::new(); let mut data = Vec::new();
while let Some(obj) = self.objects.get(self.next) { while let Some(obj) = self.objects.get(self.next) {
if data.len() + obj.len() > capacity { if data.len() + obj.len() > capacity {
+7 -2
View File
@@ -12,6 +12,7 @@
#[cfg(not(feature = "std"))] #[cfg(not(feature = "std"))]
use alloc::{format, vec, vec::Vec}; use alloc::{format, vec, vec::Vec};
use crate::addr::to_usize;
use crate::chunked_read::{alloc_output, checked_byte_len, list_chunks}; use crate::chunked_read::{alloc_output, checked_byte_len, list_chunks};
use crate::data_layout::DataLayout; use crate::data_layout::DataLayout;
use crate::dataspace::Dataspace; use crate::dataspace::Dataspace;
@@ -256,7 +257,11 @@ pub fn apply_to_unallocated_chunks(
length_size, length_size,
)?; )?;
let rank = chunk_dims.len(); let rank = chunk_dims.len();
let ds_dims: Vec<usize> = dataspace.dimensions.iter().map(|&d| d as usize).collect(); let ds_dims: Vec<usize> = dataspace
.dimensions
.iter()
.map(|&d| to_usize(d))
.collect::<Result<_, _>>()?;
if rank == 0 || ds_dims.len() != rank || chunk_dims.contains(&0) { if rank == 0 || ds_dims.len() != rank || chunk_dims.contains(&0) {
return Ok(()); return Ok(());
} }
@@ -288,7 +293,7 @@ pub fn apply_to_unallocated_chunks(
let mut cell = 0usize; let mut cell = 0usize;
let mut in_range = true; let mut in_range = true;
for d in 0..rank { for d in 0..rank {
let coord = chunk.offsets[d] as usize / chunk_dims[d]; let coord = to_usize(chunk.offsets[d])? / chunk_dims[d];
if coord >= grid[d] { if coord >= grid[d] {
in_range = false; in_range = false;
break; break;
+13 -3
View File
@@ -3,6 +3,8 @@
#[cfg(not(feature = "std"))] #[cfg(not(feature = "std"))]
extern crate alloc; extern crate alloc;
#[cfg(feature = "deflate")]
use crate::addr::saturating_usize;
#[cfg(not(feature = "std"))] #[cfg(not(feature = "std"))]
use alloc::{boxed::Box, format, vec, vec::Vec}; use alloc::{boxed::Box, format, vec, vec::Vec};
@@ -1114,7 +1116,11 @@ fn inflate_bounded_into(
loop { loop {
let (in_before, out_before) = (inflater.total_in(), inflater.total_out()); let (in_before, out_before) = (inflater.total_in(), inflater.total_out());
let status = inflater let status = inflater
.decompress_vec(&data[in_before as usize..], out, FlushDecompress::Finish) .decompress_vec(
&data[saturating_usize(in_before)..],
out,
FlushDecompress::Finish,
)
.map_err(|e| format!("deflate: {e}"))?; .map_err(|e| format!("deflate: {e}"))?;
if out.len() > limit { if out.len() > limit {
return Err("deflate: output exceeds size limit".into()); return Err("deflate: output exceeds size limit".into());
@@ -1132,7 +1138,7 @@ fn inflate_bounded_into(
} }
Status::Ok | Status::BufError => { Status::Ok | Status::BufError => {
// Room left, so the decoder stopped for want of input. // Room left, so the decoder stopped for want of input.
if inflater.total_in() as usize >= data.len() if saturating_usize(inflater.total_in()) >= data.len()
|| (inflater.total_in(), inflater.total_out()) == (in_before, out_before) || (inflater.total_in(), inflater.total_out()) == (in_before, out_before)
{ {
return Err("deflate: truncated stream".into()); return Err("deflate: truncated stream".into());
@@ -1232,7 +1238,11 @@ pub(crate) fn deflate_bounded(data: &[u8], level: u32) -> Result<Vec<u8>, String
loop { loop {
let (in_before, out_before) = (deflater.total_in(), deflater.total_out()); let (in_before, out_before) = (deflater.total_in(), deflater.total_out());
let status = deflater let status = deflater
.compress_vec(&data[in_before as usize..], &mut out, FlushCompress::Finish) .compress_vec(
&data[saturating_usize(in_before)..],
&mut out,
FlushCompress::Finish,
)
.map_err(|e| format!("deflate: {e}"))?; .map_err(|e| format!("deflate: {e}"))?;
match status { match status {
Status::StreamEnd => return Ok(out), Status::StreamEnd => return Ok(out),
+2 -1
View File
@@ -46,6 +46,7 @@
//! variable-length blocks, dictionaries, lazy chunks, user-defined codecs //! variable-length blocks, dictionaries, lazy chunks, user-defined codecs
//! and registered filters (e.g. bytedelta), sparse frames. //! and registered filters (e.g. bytedelta), sparse frames.
use crate::addr::saturating_usize;
use crate::error::FormatError; use crate::error::FormatError;
use crate::filter_registry::FilterContext; use crate::filter_registry::FilterContext;
use crate::filters_bitshuffle::bitunshuffle_block; use crate::filters_bitshuffle::bitunshuffle_block;
@@ -546,7 +547,7 @@ fn parse_frame(buf: &[u8], limit: usize) -> Result<Frame<'_>, FormatError> {
return Err(err("negative size in frame header")); return Err(err("negative size in frame header"));
} }
let header_len = header_len as usize; let header_len = header_len as usize;
let buf = &buf[..frame_len as usize]; let buf = &buf[..saturating_usize(frame_len)];
let cbytes = usize::try_from(cbytes).map_err(|_| err("bad compressed size"))?; let cbytes = usize::try_from(cbytes).map_err(|_| err("bad compressed size"))?;
let data_end = header_len let data_end = header_len
.checked_add(cbytes) .checked_add(cbytes)
+4 -3
View File
@@ -4,6 +4,7 @@
//! the compression level). Decoded with the `bzip2` crate's default backend, //! the compression level). Decoded with the `bzip2` crate's default backend,
//! `libbz2-rs-sys`, a pure-Rust port of libbzip2. //! `libbz2-rs-sys`, a pure-Rust port of libbzip2.
use crate::addr::saturating_usize;
use crate::error::FormatError; use crate::error::FormatError;
use crate::filter_registry::FilterContext; use crate::filter_registry::FilterContext;
@@ -28,7 +29,7 @@ pub(crate) fn bzip2_decode(input: &[u8], ctx: &FilterContext<'_>) -> Result<Vec<
loop { loop {
let (in_before, out_before) = (dec.total_in(), dec.total_out()); let (in_before, out_before) = (dec.total_in(), dec.total_out());
let status = dec let status = dec
.decompress_vec(&input[in_before as usize..], &mut out) .decompress_vec(&input[saturating_usize(in_before)..], &mut out)
.map_err(|e| err(&e.to_string()))?; .map_err(|e| err(&e.to_string()))?;
if out.len() > limit { if out.len() > limit {
return Err(err("output exceeds the chunk size")); return Err(err("output exceeds the chunk size"));
@@ -43,7 +44,7 @@ pub(crate) fn bzip2_decode(input: &[u8], ctx: &FilterContext<'_>) -> Result<Vec<
.max(1); .max(1);
out.try_reserve_exact(grow) out.try_reserve_exact(grow)
.map_err(|_| err("cannot allocate the output buffer"))?; .map_err(|_| err("cannot allocate the output buffer"))?;
} else if dec.total_in() as usize >= input.len() } else if saturating_usize(dec.total_in()) >= input.len()
|| (dec.total_in(), dec.total_out()) == (in_before, out_before) || (dec.total_in(), dec.total_out()) == (in_before, out_before)
{ {
return Err(err("truncated stream")); return Err(err("truncated stream"));
@@ -61,7 +62,7 @@ pub(crate) fn bzip2_encode(input: &[u8], ctx: &FilterContext<'_>) -> Result<Vec<
// bzip2's worst case is about 1% + 600 bytes over the input. // bzip2's worst case is about 1% + 600 bytes over the input.
let mut out = Vec::with_capacity(input.len() + input.len() / 100 + 600); let mut out = Vec::with_capacity(input.len() + input.len() / 100 + 600);
loop { loop {
let consumed = enc.total_in() as usize; let consumed = saturating_usize(enc.total_in());
let status = enc let status = enc
.compress_vec(&input[consumed..], &mut out, Action::Finish) .compress_vec(&input[consumed..], &mut out, Action::Finish)
.map_err(|e| cerr(e.to_string()))?; .map_err(|e| cerr(e.to_string()))?;
+3 -2
View File
@@ -6,6 +6,7 @@ extern crate alloc;
#[cfg(not(feature = "std"))] #[cfg(not(feature = "std"))]
use alloc::{format, vec, vec::Vec}; use alloc::{format, vec, vec::Vec};
use crate::addr::to_usize;
use crate::chunk_grid::ChunkGrid; use crate::chunk_grid::ChunkGrid;
use crate::chunked_read::ChunkInfo; use crate::chunked_read::ChunkInfo;
use crate::error::FormatError; use crate::error::FormatError;
@@ -158,7 +159,7 @@ pub fn read_fixed_array_chunks(
offset_size: u8, offset_size: u8,
_length_size: u8, _length_size: u8,
) -> Result<Vec<ChunkInfo>, FormatError> { ) -> Result<Vec<ChunkInfo>, FormatError> {
let db_offset = header.data_block_address as usize; let db_offset = to_usize(header.data_block_address)?;
// Parse data block header: FADB(4) + version(1) + client_id(1) + header_address(offset_size) // Parse data block header: FADB(4) + version(1) + client_id(1) + header_address(offset_size)
let db_header_size = 4 + 1 + 1 + offset_size as usize; let db_header_size = 4 + 1 + 1 + offset_size as usize;
@@ -174,7 +175,7 @@ pub fn read_fixed_array_chunks(
// Elements start immediately after the data block prefix. // Elements start immediately after the data block prefix.
let elements_start = db_offset + db_header_size; let elements_start = db_offset + db_header_size;
let num_elements = header.num_elements as usize; let num_elements = to_usize(header.num_elements)?;
// A chunk index cannot describe more elements than the file has bytes (each // A chunk index cannot describe more elements than the file has bytes (each
// element occupies at least `offset_size` bytes). Reject a corrupt count // element occupies at least `offset_size` bytes). Reject a corrupt count
// before it can drive a huge loop or overflow an offset computation. // before it can drive a huge loop or overflow an offset computation.
+6 -5
View File
@@ -3,6 +3,7 @@
#[cfg(not(feature = "std"))] #[cfg(not(feature = "std"))]
use alloc::{string::String, vec::Vec}; use alloc::{string::String, vec::Vec};
use crate::addr::to_usize;
use crate::btree_v1::collect_symbol_table_nodes; use crate::btree_v1::collect_symbol_table_nodes;
use crate::error::FormatError; use crate::error::FormatError;
use crate::local_heap::LocalHeap; use crate::local_heap::LocalHeap;
@@ -54,7 +55,7 @@ pub(crate) fn v1_group_entries(
// Parse local heap // Parse local heap
let heap = LocalHeap::parse( let heap = LocalHeap::parse(
file_data, file_data,
sym_table_msg.local_heap_address as usize, to_usize(sym_table_msg.local_heap_address)?,
offset_size, offset_size,
length_size, length_size,
)?; )?;
@@ -70,7 +71,7 @@ pub(crate) fn v1_group_entries(
let mut entries = Vec::new(); let mut entries = Vec::new();
let mut heap_checked = false; let mut heap_checked = false;
for snod_addr in snod_addrs { for snod_addr in snod_addrs {
let snod = SymbolTableNode::parse(file_data, snod_addr as usize, offset_size)?; let snod = SymbolTableNode::parse(file_data, to_usize(snod_addr)?, offset_size)?;
for entry in &snod.entries { for entry in &snod.entries {
// Like libhdf5, look at the heap's free list only once a name is // Like libhdf5, look at the heap's free list only once a name is
// needed: an empty group with a damaged heap still lists. // needed: an empty group with a damaged heap still lists.
@@ -152,7 +153,7 @@ fn for_each_v1_soft_link(
) -> Result<(), FormatError> { ) -> Result<(), FormatError> {
let heap = LocalHeap::parse( let heap = LocalHeap::parse(
file_data, file_data,
sym_table_msg.local_heap_address as usize, to_usize(sym_table_msg.local_heap_address)?,
offset_size, offset_size,
length_size, length_size,
)?; )?;
@@ -164,7 +165,7 @@ fn for_each_v1_soft_link(
)?; )?;
let mut heap_checked = false; let mut heap_checked = false;
for snod_addr in snod_addrs { for snod_addr in snod_addrs {
let snod = SymbolTableNode::parse(file_data, snod_addr as usize, offset_size)?; let snod = SymbolTableNode::parse(file_data, to_usize(snod_addr)?, offset_size)?;
for entry in &snod.entries { for entry in &snod.entries {
if entry.cache_type != CACHE_TYPE_SOFT_LINK { if entry.cache_type != CACHE_TYPE_SOFT_LINK {
continue; continue;
@@ -242,7 +243,7 @@ pub fn resolve_path(
// Not last — must be a group, parse its object header to get symbol table // Not last — must be a group, parse its object header to get symbol table
let obj_header = ObjectHeader::parse( let obj_header = ObjectHeader::parse(
file_data, file_data,
entry.object_header_address as usize, to_usize(entry.object_header_address)?,
offset_size, offset_size,
length_size, length_size,
)?; )?;
+2 -1
View File
@@ -3,6 +3,7 @@
#[cfg(not(feature = "std"))] #[cfg(not(feature = "std"))]
use alloc::{string::String, vec::Vec}; use alloc::{string::String, vec::Vec};
use crate::addr::to_usize;
use crate::datatype::CharacterSet; use crate::datatype::CharacterSet;
use crate::error::FormatError; use crate::error::FormatError;
@@ -247,7 +248,7 @@ impl LinkMessage {
}; };
// Link name length // Link name length
let name_len = read_offset(data, pos, name_size_field_width)? as usize; let name_len = to_usize(read_offset(data, pos, name_size_field_width)?)?;
pos += name_size_field_width as usize; pos += name_size_field_width as usize;
// Link name // Link name
+4 -3
View File
@@ -3,6 +3,7 @@
#[cfg(not(feature = "std"))] #[cfg(not(feature = "std"))]
use alloc::string::String; use alloc::string::String;
use crate::addr::to_usize;
use crate::error::FormatError; use crate::error::FormatError;
/// Parsed HDF5 Local Heap header. /// Parsed HDF5 Local Heap header.
@@ -140,15 +141,15 @@ impl LocalHeap {
/// Read a null-terminated string from the heap's data segment at the given byte offset. /// Read a null-terminated string from the heap's data segment at the given byte offset.
pub fn read_string(&self, file_data: &[u8], string_offset: u64) -> Result<String, FormatError> { pub fn read_string(&self, file_data: &[u8], string_offset: u64) -> Result<String, FormatError> {
let seg_addr = self.data_segment_address as usize; let seg_addr = to_usize(self.data_segment_address)?;
let str_start = let str_start =
seg_addr seg_addr
.checked_add(string_offset as usize) .checked_add(to_usize(string_offset)?)
.ok_or(FormatError::Overflow( .ok_or(FormatError::Overflow(
"local heap seg_addr + string_offset overflow".into(), "local heap seg_addr + string_offset overflow".into(),
))?; ))?;
let seg_end = seg_addr let seg_end = seg_addr
.checked_add(self.data_segment_size as usize) .checked_add(to_usize(self.data_segment_size)?)
.ok_or(FormatError::Overflow( .ok_or(FormatError::Overflow(
"local heap seg_addr + data_segment_size overflow".into(), "local heap seg_addr + data_segment_size overflow".into(),
))?; ))?;
+6 -5
View File
@@ -5,6 +5,7 @@ use alloc::vec::Vec;
use byteorder::{ByteOrder, LittleEndian}; use byteorder::{ByteOrder, LittleEndian};
use crate::addr::to_usize;
use crate::error::FormatError; use crate::error::FormatError;
use crate::message_type::MessageType; use crate::message_type::MessageType;
@@ -264,8 +265,8 @@ impl ObjectHeader {
// Follow continuations (v1 continuation chunks are just raw // Follow continuations (v1 continuation chunks are just raw
// messages, no signature); check_message has checked the body. // messages, no signature); check_message has checked the body.
if msg_type == MessageType::ObjectHeaderContinuation { if msg_type == MessageType::ObjectHeaderContinuation {
let cont_offset = read_offset(body, 0, offset_size)? as usize; let cont_offset = to_usize(read_offset(body, 0, offset_size)?)?;
let cont_length = read_offset(body, offset_size as usize, length_size)? as usize; let cont_length = to_usize(read_offset(body, offset_size as usize, length_size)?)?;
Self::parse_v1_chunk( Self::parse_v1_chunk(
data, data,
cont_offset, cont_offset,
@@ -339,7 +340,7 @@ impl ObjectHeader {
_ => unreachable!(), _ => unreachable!(),
}; };
ensure_len(data, pos, chunk_size_width as usize)?; ensure_len(data, pos, chunk_size_width as usize)?;
let chunk0_size = read_offset(data, pos, chunk_size_width)? as usize; let chunk0_size = to_usize(read_offset(data, pos, chunk_size_width)?)?;
pos += chunk_size_width as usize; pos += chunk_size_width as usize;
// Bit 2: attribute creation order tracked → messages include creation order field // Bit 2: attribute creation order tracked → messages include creation order field
let has_creation_order = flags & 0x04 != 0; let has_creation_order = flags & 0x04 != 0;
@@ -472,8 +473,8 @@ impl ObjectHeader {
let msg_type = MessageType::from_u16(msg_type_raw); let msg_type = MessageType::from_u16(msg_type_raw);
if msg_type == MessageType::ObjectHeaderContinuation { if msg_type == MessageType::ObjectHeaderContinuation {
// check_message has checked the body holds both fields. // check_message has checked the body holds both fields.
let cont_off = read_offset(body, 0, offset_size)? as usize; let cont_off = to_usize(read_offset(body, 0, offset_size)?)?;
let cont_len = read_offset(body, offset_size as usize, length_size)? as usize; let cont_len = to_usize(read_offset(body, offset_size as usize, length_size)?)?;
continuations.push((cont_off, cont_len)); continuations.push((cont_off, cont_len));
} else if msg_type == MessageType::Nil { } else if msg_type == MessageType::Nil {
null_count += 1; null_count += 1;
+12 -3
View File
@@ -203,7 +203,12 @@ fn copy_overlap(
}; };
let (src_strides, out_strides) = (strides(src_shape), strides(box_extent)); let (src_strides, out_strides) = (strides(src_shape), strides(box_extent));
let last = rank - 1; let last = rank - 1;
let run = ((hi[last] - lo[last]) as usize) * elem_size; // Byte offsets into the in-memory buffers; one that does not fit `usize`
// (a 32-bit target) is out of both buffers, like one past their ends.
let bytes = |elements: u64| usize::try_from(elements).ok()?.checked_mul(elem_size);
let Some(run) = bytes(hi[last] - lo[last]) else {
return;
};
let mut idx = lo.clone(); let mut idx = lo.clone();
loop { loop {
@@ -213,8 +218,12 @@ fn copy_overlap(
let out_at: u64 = (0..rank) let out_at: u64 = (0..rank)
.map(|d| (idx[d] - box_start[d]) * out_strides[d]) .map(|d| (idx[d] - box_start[d]) * out_strides[d])
.sum(); .sum();
let (s, o) = (src_at as usize * elem_size, out_at as usize * elem_size); if let (Some(s), Some(o)) = (bytes(src_at), bytes(out_at))
if let (Some(from), Some(to)) = (src.get(s..s + run), out.get_mut(o..o + run)) { && let (Some(from), Some(to)) = (
src.get(s..s.saturating_add(run)),
out.get_mut(o..o.saturating_add(run)),
)
{
to.copy_from_slice(from); to.copy_from_slice(from);
} }
// Advance over every dimension but the last. // Advance over every dimension but the last.
+5 -4
View File
@@ -19,6 +19,7 @@ use alloc::{vec, vec::Vec};
use core::ops::Range; use core::ops::Range;
use crate::addr::to_usize;
use crate::error::FormatError; use crate::error::FormatError;
/// A selection describing which elements of a dataset to access. /// A selection describing which elements of a dataset to access.
@@ -562,7 +563,7 @@ fn decode_hyperslab(r: &mut SelReader, version: u64) -> Result<SerializedSelecti
if !matches!(enc_size, 2 | 4 | 8) { if !matches!(enc_size, 2 | 4 | 8) {
return Err(sel_err("unsupported hyperslab coordinate encoding size")); return Err(sel_err("unsupported hyperslab coordinate encoding size"));
} }
let rank = r.uint(4)? as usize; let rank = to_usize(r.uint(4)?)?;
// HDF5 caps dataspace rank at 32 (H5S_MAX_RANK). Reject anything else so a // HDF5 caps dataspace rank at 32 (H5S_MAX_RANK). Reject anything else so a
// corrupt rank can't drive a huge allocation or read loop. // corrupt rank can't drive a huge allocation or read loop.
if rank == 0 || rank > 32 { if rank == 0 || rank > 32 {
@@ -625,11 +626,11 @@ fn decode_hyperslab(r: &mut SelReader, version: u64) -> Result<SerializedSelecti
return Err(FormatError::UnexpectedEof { return Err(FormatError::UnexpectedEof {
expected: r expected: r
.pos .pos
.saturating_add(nblocks.saturating_mul(per_block) as usize), .saturating_add(to_usize(nblocks.saturating_mul(per_block))?),
available: r.data.len(), available: r.data.len(),
}); });
} }
let n = nblocks as usize * rank; let n = to_usize(nblocks)? * rank;
let (mut starts, mut ends) = (Vec::with_capacity(n), Vec::with_capacity(n)); let (mut starts, mut ends) = (Vec::with_capacity(n), Vec::with_capacity(n));
for _ in 0..nblocks { for _ in 0..nblocks {
for _ in 0..rank { for _ in 0..rank {
@@ -662,7 +663,7 @@ fn blocks_union_coords(
.filter(|&t| t <= MAX_EXPANDED_POINTS) .filter(|&t| t <= MAX_EXPANDED_POINTS)
.ok_or_else(|| sel_err("irregular hyperslab selection is too large to expand"))?; .ok_or_else(|| sel_err("irregular hyperslab selection is too large to expand"))?;
} }
let mut out = Vec::with_capacity(total as usize); let mut out = Vec::with_capacity(to_usize(total)?);
for (s, e) in starts.chunks_exact(rank).zip(ends.chunks_exact(rank)) { for (s, e) in starts.chunks_exact(rank).zip(ends.chunks_exact(rank)) {
let mut cur = s.to_vec(); let mut cur = s.to_vec();
'block: loop { 'block: loop {
+5 -4
View File
@@ -23,6 +23,7 @@ use alloc::vec::Vec;
#[cfg(feature = "std")] #[cfg(feature = "std")]
use std::borrow::Cow; use std::borrow::Cow;
use crate::addr::to_usize;
use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records}; use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records};
use crate::error::FormatError; use crate::error::FormatError;
use crate::fractal_heap::FractalHeapHeader; use crate::fractal_heap::FractalHeapHeader;
@@ -421,7 +422,7 @@ pub fn load_sohm_table(
else { else {
return Ok(None); return Ok(None);
}; };
let ext = ObjectHeader::parse(file_data, ext_addr as usize, offset_size, length_size)?; let ext = ObjectHeader::parse(file_data, to_usize(ext_addr)?, offset_size, length_size)?;
let Some(msg) = ext let Some(msg) = ext
.messages .messages
.iter() .iter()
@@ -432,7 +433,7 @@ pub fn load_sohm_table(
let table_msg = parse_sohm_table_message(&msg.data, offset_size)?; let table_msg = parse_sohm_table_message(&msg.data, offset_size)?;
parse_sohm_table( parse_sohm_table(
file_data, file_data,
table_msg.table_address as usize, to_usize(table_msg.table_address)?,
table_msg.nindexes, table_msg.nindexes,
offset_size, offset_size,
) )
@@ -505,7 +506,7 @@ pub fn resolve_sohm_message(
let fh_header = FractalHeapHeader::parse( let fh_header = FractalHeapHeader::parse(
file_data, file_data,
index.heap_addr as usize, to_usize(index.heap_addr)?,
offset_size, offset_size,
length_size, length_size,
)?; )?;
@@ -587,7 +588,7 @@ pub fn resolve_shared_message_with_sohm(
) { ) {
(Some(addr), _) => { (Some(addr), _) => {
let target_header = let target_header =
ObjectHeader::parse(file_data, addr as usize, offset_size, length_size)?; ObjectHeader::parse(file_data, to_usize(addr)?, offset_size, length_size)?;
for msg in &target_header.messages { for msg in &target_header.messages {
if msg.msg_type == target_msg_type && !is_shared(msg.flags) { if msg.msg_type == target_msg_type && !is_shared(msg.flags) {
return Ok(msg.data.clone()); return Ok(msg.data.clone());
+7 -6
View File
@@ -15,6 +15,7 @@
#[cfg(not(feature = "std"))] #[cfg(not(feature = "std"))]
use alloc::{format, string::String, vec, vec::Vec}; use alloc::{format, string::String, vec, vec::Vec};
use crate::addr::to_usize;
use crate::data_layout::{DataLayout, VdsMapping, parse_vds_mappings}; use crate::data_layout::{DataLayout, VdsMapping, parse_vds_mappings};
use crate::dataspace::Dataspace; use crate::dataspace::Dataspace;
use crate::datatype::Datatype; use crate::datatype::Datatype;
@@ -208,7 +209,7 @@ fn load_mappings(
return Ok(Vec::new()); return Ok(Vec::new());
}; };
let coll = let coll =
crate::global_heap::GlobalHeapCollection::parse(file_data, addr as usize, length_size)?; crate::global_heap::GlobalHeapCollection::parse(file_data, to_usize(addr)?, length_size)?;
let index = u16::try_from(*global_heap_index) let index = u16::try_from(*global_heap_index)
.map_err(|_| vds_err("VDS mapping heap index out of range"))?; .map_err(|_| vds_err("VDS mapping heap index out of range"))?;
let obj = coll let obj = coll
@@ -611,12 +612,12 @@ fn scatter(
return Err(vds_err("virtual/source selection element counts differ")); return Err(vds_err("virtual/source selection element counts differ"));
} }
for (&v, &s) in vidx.iter().zip(sidx) { for (&v, &s) in vidx.iter().zip(sidx) {
let (vo, so) = (v as usize * elem_size, s as usize * elem_size); let (vo, so) = (to_usize(v)? * elem_size, to_usize(s)? * elem_size);
if vo + elem_size > out.len() || so + elem_size > src.len() { if vo + elem_size > out.len() || so + elem_size > src.len() {
return Err(vds_err("virtual dataset selection out of bounds")); return Err(vds_err("virtual dataset selection out of bounds"));
} }
out[vo..vo + elem_size].copy_from_slice(&src[so..so + elem_size]); out[vo..vo + elem_size].copy_from_slice(&src[so..so + elem_size]);
mapped[v as usize] = true; mapped[to_usize(v)?] = true;
} }
Ok(()) Ok(())
} }
@@ -747,7 +748,7 @@ fn selection_indices(
return Err(vds_err("VDS selection blocks overlap")); return Err(vds_err("VDS selection blocks overlap"));
} }
} }
let mut out = Vec::with_capacity(volume as usize); let mut out = Vec::with_capacity(to_usize(volume)?);
for (s, e) in starts.chunks_exact(rank).zip(ends.chunks_exact(rank)) { for (s, e) in starts.chunks_exact(rank).zip(ends.chunks_exact(rank)) {
let mut cur = s.to_vec(); let mut cur = s.to_vec();
'block: loop { 'block: loop {
@@ -868,7 +869,7 @@ fn load_source_file(whole: &mut [u8]) -> Result<(), FormatError> {
// read as before, up to its length. // read as before, up to its length.
let end = sb let end = sb
.data_end(base as u64, whole.len() as u64) .data_end(base as u64, whole.len() as u64)
.map_or(whole.len(), |e| base + e as usize); .map_or(Ok(whole.len()), |e| to_usize(e).map(|e| base + e))?;
crate::superblock_ext::apply_cache_image_in_place(&mut whole[base..end], &sb) crate::superblock_ext::apply_cache_image_in_place(&mut whole[base..end], &sb)
} }
@@ -921,7 +922,7 @@ fn open_source(file_data: &[u8], path: &str) -> Result<Option<OpenSource>, Forma
Err(FormatError::PathNotFound(_)) => return Ok(None), Err(FormatError::PathNotFound(_)) => return Ok(None),
Err(e) => return Err(e), Err(e) => return Err(e),
}; };
let header = crate::object_header::ObjectHeader::parse(file_data, addr as usize, os, ls)?; let header = crate::object_header::ObjectHeader::parse(file_data, to_usize(addr)?, os, ls)?;
let mut src = OpenSource { let mut src = OpenSource {
offset_size: os, offset_size: os,
length_size: ls, length_size: ls,
+4 -3
View File
@@ -9,6 +9,7 @@ use alloc::{collections::BTreeMap, format, string::String, vec, vec::Vec};
#[cfg(feature = "std")] #[cfg(feature = "std")]
use std::collections::BTreeMap; use std::collections::BTreeMap;
use crate::addr::to_usize;
use crate::error::FormatError; use crate::error::FormatError;
use crate::global_heap::{GlobalHeapCollection, GlobalHeapIndex}; use crate::global_heap::{GlobalHeapCollection, GlobalHeapIndex};
@@ -55,7 +56,7 @@ pub fn parse_vl_references(
) -> Result<Vec<VlElement>, FormatError> { ) -> Result<Vec<VlElement>, FormatError> {
let elem_size = 4 + offset_size as usize + 4; // length + address + index let elem_size = 4 + offset_size as usize + 4; // length + address + index
let total = let total =
(num_elements as usize) to_usize(num_elements)?
.checked_mul(elem_size) .checked_mul(elem_size)
.ok_or(FormatError::UnexpectedEof { .ok_or(FormatError::UnexpectedEof {
expected: usize::MAX, expected: usize::MAX,
@@ -68,7 +69,7 @@ pub fn parse_vl_references(
}); });
} }
let mut elements = Vec::with_capacity(num_elements as usize); let mut elements = Vec::with_capacity(to_usize(num_elements)?);
let mut pos = 0; let mut pos = 0;
for _ in 0..num_elements { for _ in 0..num_elements {
@@ -406,7 +407,7 @@ impl<'a> VlResolver<'a> {
let index = let index =
GlobalHeapCollection::parse_index(self.file_data, offset, self.length_size)?; GlobalHeapCollection::parse_index(self.file_data, offset, self.length_size)?;
// parse_index checked that the collection lies in the file. // parse_index checked that the collection lies in the file.
let end = offset + index.collection_size as usize; let end = offset + to_usize(index.collection_size)?;
self.check_overlap(offset, end)?; self.check_overlap(offset, end)?;
let coll = CachedCollection::new(index); let coll = CachedCollection::new(index);
if self.cached_bytes.saturating_add(coll.cost()) > self.budget { if self.cached_bytes.saturating_add(coll.cost()) > self.budget {
+35
View File
@@ -0,0 +1,35 @@
#!/usr/bin/env bash
# CI check: clawhdf5-format has no truncating `u64 as usize` cast on a 32-bit
# target. HDF5 addresses and lengths are 64-bit; on wasm32 (or any 32-bit
# target) such a cast silently wraps an address past 4 GiB onto another part
# of the file. File values go through `addr::to_usize` (a clean error) and
# in-memory counts through `addr::saturating_usize`.
#
# Lints wasm32-unknown-unknown with clippy's cast_possible_truncation and
# fails on any u64 -> usize finding (other truncations are not checked here).
#
# Usage:
# ./scripts/check-32bit-casts.sh
#
# Prerequisites:
# rustup target add wasm32-unknown-unknown
set -euo pipefail
TARGET="wasm32-unknown-unknown"
echo "==> Checking for truncating u64 -> usize casts in clawhdf5-format ($TARGET)"
out=$(cargo clippy -p clawhdf5-format --target "$TARGET" \
--features plugin-filters --message-format short \
-- -A clippy::all -W clippy::cast_possible_truncation 2>&1) || {
echo "$out"
echo "==> clippy failed" >&2
exit 1
}
found=$(grep -F 'casting `u64` to `usize`' <<<"$out" || true)
if [ -n "$found" ]; then
echo "$found"
echo "==> use addr::to_usize (file values) or addr::saturating_usize (in-memory counts)" >&2
exit 1
fi
echo "==> no truncating u64 -> usize casts"
+2
View File
@@ -147,6 +147,8 @@ run_step "wasm32 clippy (clawhdf5-wasm)" cargo clippy \
--target wasm32-unknown-unknown \ --target wasm32-unknown-unknown \
--all-targets \ --all-targets \
-- -D warnings -- -D warnings
# A 64-bit file address must not wrap on a 32-bit target.
run_step "check-32bit-casts.sh" "$SCRIPT_DIR/check-32bit-casts.sh"
# The built wasm package, run under Node against h5py/netCDF4-written files, # The built wasm package, run under Node against h5py/netCDF4-written files,
# and the viewer page in headless Chromium when one is found. # and the viewer page in headless Chromium when one is found.