Range reads M0/M1 (indexed lookups, Storage trait), ZFP, in-place editing #17
@@ -23,6 +23,19 @@ pub fn to_usize(value: u64) -> Result<usize, FormatError> {
|
||||
usize::try_from(value).map_err(|_| too_large(value))
|
||||
}
|
||||
|
||||
/// A count or offset into an in-memory buffer (a codec's progress counter,
|
||||
/// a size the writer computed from data it holds) as a `usize`, saturating
|
||||
/// at `usize::MAX` instead of truncating.
|
||||
///
|
||||
/// For values that are bounded by the length of something in memory, so
|
||||
/// always fit; if one ever did not, a saturated index fails its bounds check
|
||||
/// or allocation instead of silently addressing the wrong bytes. A value
|
||||
/// read from the file uses [`to_usize`].
|
||||
#[inline]
|
||||
pub fn saturating_usize(value: u64) -> usize {
|
||||
usize::try_from(value).unwrap_or(usize::MAX)
|
||||
}
|
||||
|
||||
#[cold]
|
||||
#[inline(never)]
|
||||
fn too_large(value: u64) -> FormatError {
|
||||
@@ -42,6 +55,15 @@ mod tests {
|
||||
assert_eq!(to_usize(usize::MAX as u64), Ok(usize::MAX));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn saturating_conversion_never_wraps() {
|
||||
assert_eq!(saturating_usize(0), 0);
|
||||
assert_eq!(saturating_usize(0x1234), 0x1234);
|
||||
assert_eq!(saturating_usize(usize::MAX as u64), usize::MAX);
|
||||
// Past usize::MAX (32-bit targets) or at u64::MAX: saturates.
|
||||
assert_eq!(saturating_usize(u64::MAX), usize::MAX);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn values_past_usize_max_are_an_error_not_truncated() {
|
||||
// Only reachable where usize is narrower than u64; on a 64-bit host
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
#[cfg(not(feature = "std"))]
|
||||
use alloc::vec::Vec;
|
||||
|
||||
use crate::addr::to_usize;
|
||||
use crate::error::FormatError;
|
||||
|
||||
/// A parsed B-tree v1 node.
|
||||
@@ -164,7 +165,12 @@ fn collect_symbol_table_nodes_inner(
|
||||
return Err(FormatError::NestingDepthExceeded);
|
||||
}
|
||||
|
||||
let node = BTreeV1Node::parse(file_data, btree_address as usize, offset_size, length_size)?;
|
||||
let node = BTreeV1Node::parse(
|
||||
file_data,
|
||||
to_usize(btree_address)?,
|
||||
offset_size,
|
||||
length_size,
|
||||
)?;
|
||||
|
||||
if node.node_type != 0 {
|
||||
return Err(FormatError::InvalidBTreeNodeType(node.node_type));
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
//! libhdf5 uses (`H5B2__hdr_init`) and the reader decodes pointers with, so
|
||||
//! the pointer widths the writer encodes are the ones every reader expects.
|
||||
|
||||
use crate::addr::saturating_usize;
|
||||
#[cfg(not(feature = "std"))]
|
||||
use alloc::{format, vec, vec::Vec};
|
||||
|
||||
@@ -105,7 +106,9 @@ pub(crate) fn build_btree_v2(
|
||||
first_node: addr + hdr_len as u64,
|
||||
nodes: Vec::new(),
|
||||
};
|
||||
let root = (n > 0).then(|| w.node(depth, 0, n as usize)).transpose()?;
|
||||
let root = (n > 0)
|
||||
.then(|| w.node(depth, 0, saturating_usize(n)))
|
||||
.transpose()?;
|
||||
|
||||
let mut out = Vec::with_capacity(hdr_len + w.nodes.len() * p.node_size as usize);
|
||||
out.extend_from_slice(b"BTHD");
|
||||
@@ -201,7 +204,7 @@ impl TreeWriter<'_> {
|
||||
"cannot spread {n} B-tree v2 records over {k} children at depth {depth}"
|
||||
)));
|
||||
}
|
||||
let k = k as usize;
|
||||
let k = saturating_usize(k);
|
||||
let in_children = n - (k - 1);
|
||||
let (base, extra) = (in_children / k, in_children % k);
|
||||
|
||||
|
||||
@@ -18,6 +18,7 @@ use alloc::collections::BTreeMap;
|
||||
#[cfg(feature = "std")]
|
||||
use std::collections::HashMap;
|
||||
|
||||
use crate::addr::to_usize;
|
||||
use crate::chunk_cache::ChunkCoord;
|
||||
use crate::chunked_read::ChunkInfo;
|
||||
|
||||
@@ -167,7 +168,15 @@ impl ChunkLayout {
|
||||
|
||||
for (_coord, ci) in index.iter() {
|
||||
let coord: ChunkCoord = ci.offsets.iter().take(rank).copied().collect();
|
||||
let chunk_offsets: Vec<usize> = coord.iter().map(|&o| o as usize).collect();
|
||||
// `ds_dims` are `usize`: a chunk at an offset past `usize::MAX`
|
||||
// (only on a 32-bit target) lies outside the dataset.
|
||||
let Ok(chunk_offsets) = coord
|
||||
.iter()
|
||||
.map(|&o| to_usize(o))
|
||||
.collect::<Result<Vec<usize>, _>>()
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
|
||||
let copies = if rank == 0 {
|
||||
// Scalar dataset — single copy
|
||||
|
||||
@@ -6,6 +6,7 @@ extern crate alloc;
|
||||
#[cfg(not(feature = "std"))]
|
||||
use alloc::{format, vec, vec::Vec};
|
||||
|
||||
use crate::addr::to_usize;
|
||||
#[cfg(feature = "std")]
|
||||
use crate::chunk_cache::{CacheAlignedBuffer, ChunkCache};
|
||||
use crate::data_layout::DataLayout;
|
||||
@@ -265,7 +266,7 @@ fn fill_from_chunks(
|
||||
)));
|
||||
}
|
||||
let offsets = &c.offsets[..rank];
|
||||
let c_addr = c.address as usize;
|
||||
let c_addr = to_usize(c.address)?;
|
||||
let size = c.chunk_size as usize;
|
||||
ensure_len(file_data, c_addr, size)?;
|
||||
let raw = &file_data[c_addr..c_addr + size];
|
||||
@@ -825,7 +826,7 @@ fn parse_chunk_node(
|
||||
return Err(FormatError::NestingDepthExceeded);
|
||||
}
|
||||
|
||||
let offset = btree_address as usize;
|
||||
let offset = to_usize(btree_address)?;
|
||||
let os = offset_size as usize;
|
||||
|
||||
// Parse B-tree v1 header
|
||||
@@ -945,7 +946,8 @@ pub fn generate_implicit_chunks(
|
||||
}
|
||||
let total_chunks: u64 = num_chunks_per_dim.iter().product();
|
||||
|
||||
let mut chunks = Vec::with_capacity(total_chunks as usize);
|
||||
// A capacity hint only (a count past `usize::MAX` could not be pushed).
|
||||
let mut chunks = Vec::with_capacity(usize::try_from(total_chunks).unwrap_or(0));
|
||||
for linear_idx in 0..total_chunks {
|
||||
let mut offsets = vec![0u64; rank];
|
||||
let mut remaining = linear_idx;
|
||||
@@ -993,7 +995,7 @@ fn read_btree_v2_chunks(
|
||||
use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records};
|
||||
|
||||
let bad = |what: &str| FormatError::ChunkedReadError(format!("B-tree v2 chunk index: {what}"));
|
||||
let header = BTreeV2Header::parse(file_data, addr as usize, offset_size, length_size)?;
|
||||
let header = BTreeV2Header::parse(file_data, to_usize(addr)?, offset_size, length_size)?;
|
||||
let rank = chunk_dims.len();
|
||||
let os = offset_size as usize;
|
||||
let record_size = header.record_size as usize;
|
||||
@@ -1122,7 +1124,11 @@ pub fn list_chunks(
|
||||
|
||||
// Both v3 and v4 include element size as last dim (rank+1)
|
||||
let (rank, chunk_dims) = chunk_geometry(chunk_dimensions, version, dataspace, elem_size)?;
|
||||
let ds_dims: Vec<usize> = dataspace.dimensions.iter().map(|&d| d as usize).collect();
|
||||
let ds_dims: Vec<usize> = dataspace
|
||||
.dimensions
|
||||
.iter()
|
||||
.map(|&d| to_usize(d))
|
||||
.collect::<Result<_, _>>()?;
|
||||
|
||||
// Collect chunks based on version and index type
|
||||
let mut chunks = match (version, chunk_index_type) {
|
||||
@@ -1158,7 +1164,7 @@ pub fn list_chunks(
|
||||
// Fixed Array — use spatial chunk dims only
|
||||
let spatial_chunk_dims: &[u32] = &chunk_dimensions[..rank];
|
||||
let header =
|
||||
FixedArrayHeader::parse(file_data, addr as usize, offset_size, length_size)?;
|
||||
FixedArrayHeader::parse(file_data, to_usize(addr)?, offset_size, length_size)?;
|
||||
read_fixed_array_chunks(
|
||||
file_data,
|
||||
&header,
|
||||
@@ -1174,7 +1180,7 @@ pub fn list_chunks(
|
||||
// Extensible Array — use spatial chunk dims only
|
||||
let spatial_chunk_dims: &[u32] = &chunk_dimensions[..rank];
|
||||
let header =
|
||||
ExtensibleArrayHeader::parse(file_data, addr as usize, offset_size, length_size)?;
|
||||
ExtensibleArrayHeader::parse(file_data, to_usize(addr)?, offset_size, length_size)?;
|
||||
read_extensible_array_chunks(
|
||||
file_data,
|
||||
&header,
|
||||
@@ -1349,7 +1355,11 @@ pub(crate) fn read_chunked_full<O>(
|
||||
// dimension the total is 0 even if other dimensions are huge.
|
||||
return Ok(output);
|
||||
}
|
||||
let ds_dims: Vec<usize> = dataspace.dimensions.iter().map(|&d| d as usize).collect();
|
||||
let ds_dims: Vec<usize> = dataspace
|
||||
.dimensions
|
||||
.iter()
|
||||
.map(|&d| to_usize(d))
|
||||
.collect::<Result<_, _>>()?;
|
||||
let placer = ChunkPlacer::new(&chunk_dims, &ds_dims, elem_size);
|
||||
let chunk_total_bytes = checked_chunk_byte_len(&chunk_dims, elem_size)?;
|
||||
// Chunks are cached only when the whole dataset fits: pushing a larger
|
||||
@@ -1603,7 +1613,11 @@ pub fn read_chunked_data_sweep(
|
||||
check_chunk_element_size(layout, datatype, offset_size)?;
|
||||
let elem_size = datatype.type_size() as usize;
|
||||
let (rank, chunk_dims) = chunk_geometry(chunk_dimensions, version, dataspace, elem_size)?;
|
||||
let ds_dims: Vec<usize> = dataspace.dimensions.iter().map(|&d| d as usize).collect();
|
||||
let ds_dims: Vec<usize> = dataspace
|
||||
.dimensions
|
||||
.iter()
|
||||
.map(|&d| to_usize(d))
|
||||
.collect::<Result<_, _>>()?;
|
||||
|
||||
// The per-file cache is shared across datasets (and threads); every
|
||||
// lookup is keyed by this dataset's chunk-index address, so another
|
||||
@@ -1659,7 +1673,7 @@ pub fn read_chunked_data_sweep(
|
||||
cached
|
||||
} else {
|
||||
// Decompress from file
|
||||
let c_addr = chunk_info.address as usize;
|
||||
let c_addr = to_usize(chunk_info.address)?;
|
||||
let size = chunk_info.chunk_size as usize;
|
||||
ensure_len(file_data, c_addr, size)?;
|
||||
let raw_chunk = &file_data[c_addr..c_addr + size];
|
||||
@@ -1682,8 +1696,8 @@ pub fn read_chunked_data_sweep(
|
||||
.offsets
|
||||
.iter()
|
||||
.take(rank)
|
||||
.map(|&o| o as usize)
|
||||
.collect();
|
||||
.map(|&o| to_usize(o))
|
||||
.collect::<Result<_, _>>()?;
|
||||
|
||||
if rank == 0 {
|
||||
let copy_len = decompressed.len().min(output.len());
|
||||
@@ -1743,7 +1757,11 @@ pub fn read_chunked_data_indexed(
|
||||
check_chunk_element_size(layout, datatype, offset_size)?;
|
||||
let elem_size = datatype.type_size() as usize;
|
||||
let (rank, chunk_dims) = chunk_geometry(chunk_dimensions, version, dataspace, elem_size)?;
|
||||
let ds_dims: Vec<usize> = dataspace.dimensions.iter().map(|&d| d as usize).collect();
|
||||
let ds_dims: Vec<usize> = dataspace
|
||||
.dimensions
|
||||
.iter()
|
||||
.map(|&d| to_usize(d))
|
||||
.collect::<Result<_, _>>()?;
|
||||
|
||||
// Chunk index and assembly plan for this dataset, built on first access
|
||||
// and kept per dataset (keyed by chunk-index address) in the shared cache.
|
||||
@@ -1776,7 +1794,7 @@ pub fn read_chunked_data_indexed(
|
||||
if let Some(cached) = cache.get_decompressed_in(addr, coord) {
|
||||
chunk_buffers.push(cached);
|
||||
} else {
|
||||
let c_addr = *file_offset as usize;
|
||||
let c_addr = to_usize(*file_offset)?;
|
||||
let size = *file_size as usize;
|
||||
ensure_len(file_data, c_addr, size)?;
|
||||
let raw_chunk = &file_data[c_addr..c_addr + size];
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
#[cfg(not(feature = "std"))]
|
||||
extern crate alloc;
|
||||
|
||||
use crate::addr::saturating_usize;
|
||||
#[cfg(not(feature = "std"))]
|
||||
use alloc::{format, vec, vec::Vec};
|
||||
|
||||
@@ -414,18 +415,18 @@ pub fn split_into_chunks(
|
||||
// Dataset strides (row-major)
|
||||
let mut ds_strides = vec![1usize; rank];
|
||||
for i in (0..rank.saturating_sub(1)).rev() {
|
||||
ds_strides[i] = ds_strides[i + 1] * shape[i + 1] as usize;
|
||||
ds_strides[i] = ds_strides[i + 1] * saturating_usize(shape[i + 1]);
|
||||
}
|
||||
|
||||
// Chunk strides
|
||||
let mut chunk_strides = vec![1usize; rank];
|
||||
for i in (0..rank.saturating_sub(1)).rev() {
|
||||
chunk_strides[i] = chunk_strides[i + 1] * chunk_dims[i + 1] as usize;
|
||||
chunk_strides[i] = chunk_strides[i + 1] * saturating_usize(chunk_dims[i + 1]);
|
||||
}
|
||||
|
||||
let chunk_total_elements: usize = chunk_dims.iter().map(|&d| d as usize).product();
|
||||
let chunk_total_elements: usize = chunk_dims.iter().map(|&d| saturating_usize(d)).product();
|
||||
|
||||
let mut result = Vec::with_capacity(total_chunks as usize);
|
||||
let mut result = Vec::with_capacity(saturating_usize(total_chunks));
|
||||
|
||||
for linear_idx in 0..total_chunks {
|
||||
// Convert linear index to chunk grid coordinates
|
||||
@@ -453,8 +454,8 @@ pub fn split_into_chunks(
|
||||
let coord_in_chunk = remaining_idx / chunk_strides[d];
|
||||
remaining_idx %= chunk_strides[d];
|
||||
|
||||
let global_coord = offsets[d] as usize + coord_in_chunk;
|
||||
if global_coord >= shape[d] as usize {
|
||||
let global_coord = saturating_usize(offsets[d]) + coord_in_chunk;
|
||||
if global_coord >= saturating_usize(shape[d]) {
|
||||
out_of_bounds = true;
|
||||
break;
|
||||
}
|
||||
@@ -1036,7 +1037,7 @@ impl ChunkIndexPlan {
|
||||
Ok(Self::SingleChunk)
|
||||
} else {
|
||||
let grid = ChunkGrid::fixed_array(shape, Some(max), chunk_dims)?;
|
||||
Ok(Self::FixedArray(grid, nslots as usize))
|
||||
Ok(Self::FixedArray(grid, saturating_usize(nslots)))
|
||||
}
|
||||
}
|
||||
1 => Ok(Self::ExtensibleArray(ChunkGrid::extensible_array(
|
||||
@@ -1251,7 +1252,7 @@ pub fn write_selection_to_buffer(
|
||||
let rank = dims.len();
|
||||
let mut ds_strides = vec![1usize; rank];
|
||||
for i in (0..rank.saturating_sub(1)).rev() {
|
||||
ds_strides[i] = ds_strides[i + 1] * dims[i + 1] as usize;
|
||||
ds_strides[i] = ds_strides[i + 1] * saturating_usize(dims[i + 1]);
|
||||
}
|
||||
|
||||
let mut src_offset = 0usize;
|
||||
@@ -1301,7 +1302,7 @@ pub fn write_selection_to_buffer(
|
||||
buffer,
|
||||
new_data,
|
||||
src_offset,
|
||||
current_ds_offset + coord as usize * ds_strides[d],
|
||||
current_ds_offset + saturating_usize(coord) * ds_strides[d],
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1328,14 +1329,14 @@ pub fn write_selection_to_buffer(
|
||||
let rank = dims.len();
|
||||
let mut ds_strides = vec![1usize; rank];
|
||||
for i in (0..rank.saturating_sub(1)).rev() {
|
||||
ds_strides[i] = ds_strides[i + 1] * dims[i + 1] as usize;
|
||||
ds_strides[i] = ds_strides[i + 1] * saturating_usize(dims[i + 1]);
|
||||
}
|
||||
|
||||
for (pi, pt) in pts.iter().enumerate() {
|
||||
let flat: usize = pt
|
||||
.iter()
|
||||
.zip(ds_strides.iter())
|
||||
.map(|(&p, &s)| p as usize * s)
|
||||
.map(|(&p, &s)| saturating_usize(p) * s)
|
||||
.sum();
|
||||
let dst = flat * elem_size;
|
||||
let src = pi * elem_size;
|
||||
|
||||
@@ -6,6 +6,7 @@ use alloc::{format, string::String, vec::Vec};
|
||||
#[cfg(feature = "std")]
|
||||
use std::string::String;
|
||||
|
||||
use crate::addr::to_usize;
|
||||
use crate::error::FormatError;
|
||||
|
||||
/// A single VDS (Virtual Dataset) source mapping.
|
||||
@@ -207,7 +208,7 @@ pub fn parse_vds_mappings(
|
||||
"VDS mapping shares a name with a later entry".into(),
|
||||
));
|
||||
}
|
||||
Ok(idx as usize)
|
||||
to_usize(idx)
|
||||
};
|
||||
|
||||
let source_file = if flags & VDS_SOURCE_SAME_FILE != 0 {
|
||||
@@ -320,7 +321,7 @@ impl DataLayout {
|
||||
{
|
||||
let coll = crate::global_heap::GlobalHeapCollection::parse(
|
||||
file_data,
|
||||
addr as usize,
|
||||
to_usize(addr)?,
|
||||
length_size,
|
||||
)?;
|
||||
let obj = coll.get_object(*global_heap_index as u16).ok_or(
|
||||
|
||||
@@ -6,6 +6,7 @@ use alloc::{collections::BTreeMap, format, string::String, vec, vec::Vec};
|
||||
#[cfg(feature = "std")]
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use crate::addr::to_usize;
|
||||
#[cfg(feature = "std")]
|
||||
use crate::chunk_cache::ChunkCache;
|
||||
use crate::chunked_read::read_chunked_data;
|
||||
@@ -117,7 +118,7 @@ pub fn read_raw_data_zerocopy<'a>(
|
||||
dataspace: &Dataspace,
|
||||
datatype: &Datatype,
|
||||
) -> Result<Option<&'a [u8]>, FormatError> {
|
||||
let num_elements = dataspace.num_elements() as usize;
|
||||
let num_elements = to_usize(dataspace.num_elements())?;
|
||||
let elem_size = datatype.type_size() as usize;
|
||||
let expected_size = num_elements.checked_mul(elem_size).ok_or_else(|| {
|
||||
FormatError::Overflow(format!(
|
||||
@@ -128,7 +129,7 @@ pub fn read_raw_data_zerocopy<'a>(
|
||||
match layout {
|
||||
DataLayout::Contiguous { address, size } => {
|
||||
let addr = address.ok_or(FormatError::NoDataAllocated)?;
|
||||
let addr = addr as usize;
|
||||
let addr = to_usize(addr)?;
|
||||
let sz = contiguous_read_len(*size, expected_size)?;
|
||||
ensure_len(file_data, addr, sz)?;
|
||||
Ok(Some(&file_data[addr..addr + sz]))
|
||||
@@ -219,7 +220,7 @@ fn read_raw_data_full_impl(
|
||||
length_size: u8,
|
||||
resolver: Option<&VdsSourceResolver>,
|
||||
) -> Result<Vec<u8>, FormatError> {
|
||||
let num_elements = dataspace.num_elements() as usize;
|
||||
let num_elements = to_usize(dataspace.num_elements())?;
|
||||
let elem_size = datatype.type_size() as usize;
|
||||
let expected_size = num_elements.checked_mul(elem_size).ok_or_else(|| {
|
||||
FormatError::Overflow(format!(
|
||||
@@ -239,7 +240,7 @@ fn read_raw_data_full_impl(
|
||||
}
|
||||
DataLayout::Contiguous { address, size } => {
|
||||
let addr = address.ok_or(FormatError::NoDataAllocated)?;
|
||||
let addr = addr as usize;
|
||||
let addr = to_usize(addr)?;
|
||||
let sz = contiguous_read_len(*size, expected_size)?;
|
||||
ensure_len(file_data, addr, sz)?;
|
||||
let mut out = crate::bulk_alloc::vec_for_bulk(sz);
|
||||
@@ -582,7 +583,7 @@ pub fn extract_selection_from_buffer(
|
||||
let rank = dims.len();
|
||||
let mut ds_strides = vec![1usize; rank];
|
||||
for i in (0..rank.saturating_sub(1)).rev() {
|
||||
ds_strides[i] = ds_strides[i + 1] * dims[i + 1] as usize;
|
||||
ds_strides[i] = ds_strides[i + 1] * to_usize(dims[i + 1])?;
|
||||
}
|
||||
|
||||
let mut output = Vec::with_capacity(pts.len() * elem_size);
|
||||
@@ -590,8 +591,8 @@ pub fn extract_selection_from_buffer(
|
||||
let flat: usize = pt
|
||||
.iter()
|
||||
.zip(ds_strides.iter())
|
||||
.map(|(&p, &s)| p as usize * s)
|
||||
.sum();
|
||||
.map(|(&p, &s)| Ok(to_usize(p)? * s))
|
||||
.sum::<Result<usize, FormatError>>()?;
|
||||
let src = flat * elem_size;
|
||||
if src + elem_size <= full_data.len() {
|
||||
output.extend_from_slice(&full_data[src..src + elem_size]);
|
||||
@@ -1341,7 +1342,7 @@ pub fn read_compound_fields(
|
||||
let mut fields = Vec::with_capacity(members.len());
|
||||
for m in members {
|
||||
let field_size = m.datatype.type_size() as usize;
|
||||
let offset = m.byte_offset as usize;
|
||||
let offset = to_usize(m.byte_offset)?;
|
||||
if offset
|
||||
.checked_add(field_size)
|
||||
.is_none_or(|end| end > elem_size)
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
#[cfg(not(feature = "std"))]
|
||||
extern crate alloc;
|
||||
|
||||
use crate::addr::saturating_usize;
|
||||
#[cfg(not(feature = "std"))]
|
||||
use alloc::{vec, vec::Vec};
|
||||
|
||||
@@ -247,7 +248,7 @@ pub fn build_extensible_array_at(
|
||||
|
||||
// Header (EAHD). The six statistics are, in order: super blocks, their
|
||||
// bytes, data blocks, their bytes, max index set, elements realised.
|
||||
let mut out = Vec::with_capacity((cursor - ea_base_address) as usize);
|
||||
let mut out = Vec::with_capacity(saturating_usize(cursor - ea_base_address));
|
||||
out.extend_from_slice(b"EAHD");
|
||||
out.push(0); // version
|
||||
out.push(client_id);
|
||||
|
||||
@@ -9,6 +9,7 @@ extern crate alloc;
|
||||
#[cfg(not(feature = "std"))]
|
||||
use alloc::{format, vec, vec::Vec};
|
||||
|
||||
use crate::addr::to_usize;
|
||||
use crate::chunk_grid::ChunkGrid;
|
||||
use crate::chunked_read::ChunkInfo;
|
||||
use crate::error::FormatError;
|
||||
@@ -451,7 +452,7 @@ pub fn read_extensible_array_chunks(
|
||||
// Parse index block (EAIB): signature(4) + version(1) + client_id(1)
|
||||
// + header address(offset_size), then the inline elements, then the
|
||||
// direct data block addresses, then the super block addresses.
|
||||
let ib_offset = header.index_block_address as usize;
|
||||
let ib_offset = to_usize(header.index_block_address)?;
|
||||
let ib_header_size = 4 + 1 + 1 + os;
|
||||
ensure_len(file_data, ib_offset, ib_header_size)?;
|
||||
|
||||
@@ -463,7 +464,7 @@ pub fn read_extensible_array_chunks(
|
||||
let mut pos = ib_offset + ib_header_size;
|
||||
|
||||
let mut chunks = Vec::new();
|
||||
let total_elements = header.num_elements as usize;
|
||||
let total_elements = to_usize(header.num_elements)?;
|
||||
|
||||
let dmin = header.min_dblk_nelmts as usize;
|
||||
if dmin == 0 || !dmin.is_power_of_two() {
|
||||
@@ -563,7 +564,7 @@ pub fn read_extensible_array_chunks(
|
||||
}
|
||||
chunks.extend(read_data_block_elements(
|
||||
file_data,
|
||||
addr as usize,
|
||||
to_usize(addr)?,
|
||||
dblk_nelmts,
|
||||
header,
|
||||
offset_size,
|
||||
@@ -592,7 +593,7 @@ pub fn read_extensible_array_chunks(
|
||||
if !is_undefined_addr(sb_addr, offset_size) {
|
||||
chunks.extend(read_super_block(
|
||||
file_data,
|
||||
sb_addr as usize,
|
||||
to_usize(sb_addr)?,
|
||||
ndblks,
|
||||
dblk_nelmts,
|
||||
header,
|
||||
@@ -676,7 +677,7 @@ fn read_super_block(
|
||||
if !is_undefined_addr(addr, offset_size) {
|
||||
chunks.extend(read_data_block_elements(
|
||||
file_data,
|
||||
addr as usize,
|
||||
to_usize(addr)?,
|
||||
dblk_nelmts,
|
||||
header,
|
||||
offset_size,
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
//! Produces valid HDF5 files with v3 superblock, v2 object headers,
|
||||
//! link messages, contiguous datasets, inline and dense attributes.
|
||||
|
||||
use crate::addr::saturating_usize;
|
||||
#[cfg(not(feature = "std"))]
|
||||
use alloc::{format, vec, vec::Vec};
|
||||
|
||||
@@ -336,7 +337,7 @@ pub(crate) fn build_single_block_fractal_heap(
|
||||
|
||||
// An object must fit one direct block: the writer has no huge-object
|
||||
// path, and libhdf5 cannot read an object that overruns its block.
|
||||
let max_managed = max_direct_block_size as usize - dblock_header_size;
|
||||
let max_managed = saturating_usize(max_direct_block_size) - dblock_header_size;
|
||||
if let Some(big) = serialized.iter().find(|s| s.len() > max_managed) {
|
||||
return Err(FormatError::SerializationError(format!(
|
||||
"a {}-byte message cannot go in dense storage: a fractal heap \
|
||||
@@ -392,7 +393,7 @@ pub(crate) fn build_single_block_fractal_heap(
|
||||
let dblock_addr = frhp_addr + frhp_size as u64;
|
||||
let btree_addr = dblock_addr + starting_block_size;
|
||||
|
||||
let data_space = starting_block_size as usize - dblock_header_size;
|
||||
let data_space = saturating_usize(starting_block_size) - dblock_header_size;
|
||||
let free_space = data_space - total_data_size;
|
||||
|
||||
// Build fractal heap header
|
||||
@@ -428,7 +429,7 @@ pub(crate) fn build_single_block_fractal_heap(
|
||||
debug_assert_eq!(frhp.len(), frhp_size);
|
||||
|
||||
// Build direct block: header (with checksum) + data + padding
|
||||
let mut dblock = Vec::with_capacity(starting_block_size as usize);
|
||||
let mut dblock = Vec::with_capacity(saturating_usize(starting_block_size));
|
||||
dblock.extend_from_slice(b"FHDB");
|
||||
dblock.push(0); // version
|
||||
write_offset(&mut dblock, frhp_addr, OFFSET_SIZE);
|
||||
@@ -446,12 +447,12 @@ pub(crate) fn build_single_block_fractal_heap(
|
||||
}
|
||||
|
||||
// Pad to full block size
|
||||
dblock.resize(starting_block_size as usize, 0);
|
||||
dblock.resize(saturating_usize(starting_block_size), 0);
|
||||
|
||||
// Checksum: computed over entire block with checksum field zeroed
|
||||
let dblock_checksum = crate::checksum::jenkins_lookup3(&dblock);
|
||||
dblock[cksum_pos..cksum_pos + 4].copy_from_slice(&dblock_checksum.to_le_bytes());
|
||||
debug_assert_eq!(dblock.len(), starting_block_size as usize);
|
||||
debug_assert_eq!(dblock.len(), saturating_usize(starting_block_size));
|
||||
|
||||
// Build heap IDs
|
||||
let heap_ids: Vec<Vec<u8>> = obj_offsets
|
||||
@@ -706,7 +707,7 @@ impl HeapIndirectBlock {
|
||||
let cksum_pos = out.len();
|
||||
out.extend_from_slice(&[0u8; 4]); // checksum placeholder
|
||||
out.extend_from_slice(&b.data);
|
||||
out.resize(d + b.size as usize, 0);
|
||||
out.resize(d + saturating_usize(b.size), 0);
|
||||
let cksum = crate::checksum::jenkins_lookup3(&out[d..]);
|
||||
out[cksum_pos..cksum_pos + 4].copy_from_slice(&cksum.to_le_bytes());
|
||||
child += b.size;
|
||||
@@ -740,7 +741,7 @@ impl HeapPacker<'_> {
|
||||
nrows: Option<usize>,
|
||||
) -> Result<HeapIndirectBlock, FormatError> {
|
||||
let geom = self.geom;
|
||||
let width = geom.width as usize;
|
||||
let width = saturating_usize(geom.width);
|
||||
let mut slots = Vec::new();
|
||||
let mut off = heap_offset;
|
||||
let mut row = 0usize;
|
||||
@@ -763,7 +764,8 @@ impl HeapPacker<'_> {
|
||||
// A child whose biggest direct block cannot hold the
|
||||
// next object is skipped whole, not walked.
|
||||
let biggest = geom.row_size(child_rows.min(geom.max_direct_rows()) - 1);
|
||||
if self.objects[self.next].len() > (biggest as usize - geom.dblock_header_size)
|
||||
if self.objects[self.next].len()
|
||||
> (saturating_usize(biggest) - geom.dblock_header_size)
|
||||
{
|
||||
slots.push(HeapSlot::Empty);
|
||||
off += size;
|
||||
@@ -794,7 +796,7 @@ impl HeapPacker<'_> {
|
||||
/// objects as fit; leave it unallocated if not even the next one does.
|
||||
fn fill_direct(&mut self, heap_offset: u64, size: u64) -> HeapSlot {
|
||||
let header = self.geom.dblock_header_size;
|
||||
let capacity = size as usize - header;
|
||||
let capacity = saturating_usize(size) - header;
|
||||
let mut data = Vec::new();
|
||||
while let Some(obj) = self.objects.get(self.next) {
|
||||
if data.len() + obj.len() > capacity {
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
#[cfg(not(feature = "std"))]
|
||||
use alloc::{format, vec, vec::Vec};
|
||||
|
||||
use crate::addr::to_usize;
|
||||
use crate::chunked_read::{alloc_output, checked_byte_len, list_chunks};
|
||||
use crate::data_layout::DataLayout;
|
||||
use crate::dataspace::Dataspace;
|
||||
@@ -256,7 +257,11 @@ pub fn apply_to_unallocated_chunks(
|
||||
length_size,
|
||||
)?;
|
||||
let rank = chunk_dims.len();
|
||||
let ds_dims: Vec<usize> = dataspace.dimensions.iter().map(|&d| d as usize).collect();
|
||||
let ds_dims: Vec<usize> = dataspace
|
||||
.dimensions
|
||||
.iter()
|
||||
.map(|&d| to_usize(d))
|
||||
.collect::<Result<_, _>>()?;
|
||||
if rank == 0 || ds_dims.len() != rank || chunk_dims.contains(&0) {
|
||||
return Ok(());
|
||||
}
|
||||
@@ -288,7 +293,7 @@ pub fn apply_to_unallocated_chunks(
|
||||
let mut cell = 0usize;
|
||||
let mut in_range = true;
|
||||
for d in 0..rank {
|
||||
let coord = chunk.offsets[d] as usize / chunk_dims[d];
|
||||
let coord = to_usize(chunk.offsets[d])? / chunk_dims[d];
|
||||
if coord >= grid[d] {
|
||||
in_range = false;
|
||||
break;
|
||||
|
||||
@@ -3,6 +3,8 @@
|
||||
#[cfg(not(feature = "std"))]
|
||||
extern crate alloc;
|
||||
|
||||
#[cfg(feature = "deflate")]
|
||||
use crate::addr::saturating_usize;
|
||||
#[cfg(not(feature = "std"))]
|
||||
use alloc::{boxed::Box, format, vec, vec::Vec};
|
||||
|
||||
@@ -1114,7 +1116,11 @@ fn inflate_bounded_into(
|
||||
loop {
|
||||
let (in_before, out_before) = (inflater.total_in(), inflater.total_out());
|
||||
let status = inflater
|
||||
.decompress_vec(&data[in_before as usize..], out, FlushDecompress::Finish)
|
||||
.decompress_vec(
|
||||
&data[saturating_usize(in_before)..],
|
||||
out,
|
||||
FlushDecompress::Finish,
|
||||
)
|
||||
.map_err(|e| format!("deflate: {e}"))?;
|
||||
if out.len() > limit {
|
||||
return Err("deflate: output exceeds size limit".into());
|
||||
@@ -1132,7 +1138,7 @@ fn inflate_bounded_into(
|
||||
}
|
||||
Status::Ok | Status::BufError => {
|
||||
// Room left, so the decoder stopped for want of input.
|
||||
if inflater.total_in() as usize >= data.len()
|
||||
if saturating_usize(inflater.total_in()) >= data.len()
|
||||
|| (inflater.total_in(), inflater.total_out()) == (in_before, out_before)
|
||||
{
|
||||
return Err("deflate: truncated stream".into());
|
||||
@@ -1232,7 +1238,11 @@ pub(crate) fn deflate_bounded(data: &[u8], level: u32) -> Result<Vec<u8>, String
|
||||
loop {
|
||||
let (in_before, out_before) = (deflater.total_in(), deflater.total_out());
|
||||
let status = deflater
|
||||
.compress_vec(&data[in_before as usize..], &mut out, FlushCompress::Finish)
|
||||
.compress_vec(
|
||||
&data[saturating_usize(in_before)..],
|
||||
&mut out,
|
||||
FlushCompress::Finish,
|
||||
)
|
||||
.map_err(|e| format!("deflate: {e}"))?;
|
||||
match status {
|
||||
Status::StreamEnd => return Ok(out),
|
||||
|
||||
@@ -46,6 +46,7 @@
|
||||
//! variable-length blocks, dictionaries, lazy chunks, user-defined codecs
|
||||
//! and registered filters (e.g. bytedelta), sparse frames.
|
||||
|
||||
use crate::addr::saturating_usize;
|
||||
use crate::error::FormatError;
|
||||
use crate::filter_registry::FilterContext;
|
||||
use crate::filters_bitshuffle::bitunshuffle_block;
|
||||
@@ -546,7 +547,7 @@ fn parse_frame(buf: &[u8], limit: usize) -> Result<Frame<'_>, FormatError> {
|
||||
return Err(err("negative size in frame header"));
|
||||
}
|
||||
let header_len = header_len as usize;
|
||||
let buf = &buf[..frame_len as usize];
|
||||
let buf = &buf[..saturating_usize(frame_len)];
|
||||
let cbytes = usize::try_from(cbytes).map_err(|_| err("bad compressed size"))?;
|
||||
let data_end = header_len
|
||||
.checked_add(cbytes)
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
//! the compression level). Decoded with the `bzip2` crate's default backend,
|
||||
//! `libbz2-rs-sys`, a pure-Rust port of libbzip2.
|
||||
|
||||
use crate::addr::saturating_usize;
|
||||
use crate::error::FormatError;
|
||||
use crate::filter_registry::FilterContext;
|
||||
|
||||
@@ -28,7 +29,7 @@ pub(crate) fn bzip2_decode(input: &[u8], ctx: &FilterContext<'_>) -> Result<Vec<
|
||||
loop {
|
||||
let (in_before, out_before) = (dec.total_in(), dec.total_out());
|
||||
let status = dec
|
||||
.decompress_vec(&input[in_before as usize..], &mut out)
|
||||
.decompress_vec(&input[saturating_usize(in_before)..], &mut out)
|
||||
.map_err(|e| err(&e.to_string()))?;
|
||||
if out.len() > limit {
|
||||
return Err(err("output exceeds the chunk size"));
|
||||
@@ -43,7 +44,7 @@ pub(crate) fn bzip2_decode(input: &[u8], ctx: &FilterContext<'_>) -> Result<Vec<
|
||||
.max(1);
|
||||
out.try_reserve_exact(grow)
|
||||
.map_err(|_| err("cannot allocate the output buffer"))?;
|
||||
} else if dec.total_in() as usize >= input.len()
|
||||
} else if saturating_usize(dec.total_in()) >= input.len()
|
||||
|| (dec.total_in(), dec.total_out()) == (in_before, out_before)
|
||||
{
|
||||
return Err(err("truncated stream"));
|
||||
@@ -61,7 +62,7 @@ pub(crate) fn bzip2_encode(input: &[u8], ctx: &FilterContext<'_>) -> Result<Vec<
|
||||
// bzip2's worst case is about 1% + 600 bytes over the input.
|
||||
let mut out = Vec::with_capacity(input.len() + input.len() / 100 + 600);
|
||||
loop {
|
||||
let consumed = enc.total_in() as usize;
|
||||
let consumed = saturating_usize(enc.total_in());
|
||||
let status = enc
|
||||
.compress_vec(&input[consumed..], &mut out, Action::Finish)
|
||||
.map_err(|e| cerr(e.to_string()))?;
|
||||
|
||||
@@ -6,6 +6,7 @@ extern crate alloc;
|
||||
#[cfg(not(feature = "std"))]
|
||||
use alloc::{format, vec, vec::Vec};
|
||||
|
||||
use crate::addr::to_usize;
|
||||
use crate::chunk_grid::ChunkGrid;
|
||||
use crate::chunked_read::ChunkInfo;
|
||||
use crate::error::FormatError;
|
||||
@@ -158,7 +159,7 @@ pub fn read_fixed_array_chunks(
|
||||
offset_size: u8,
|
||||
_length_size: u8,
|
||||
) -> Result<Vec<ChunkInfo>, FormatError> {
|
||||
let db_offset = header.data_block_address as usize;
|
||||
let db_offset = to_usize(header.data_block_address)?;
|
||||
|
||||
// Parse data block header: FADB(4) + version(1) + client_id(1) + header_address(offset_size)
|
||||
let db_header_size = 4 + 1 + 1 + offset_size as usize;
|
||||
@@ -174,7 +175,7 @@ pub fn read_fixed_array_chunks(
|
||||
// Elements start immediately after the data block prefix.
|
||||
let elements_start = db_offset + db_header_size;
|
||||
|
||||
let num_elements = header.num_elements as usize;
|
||||
let num_elements = to_usize(header.num_elements)?;
|
||||
// A chunk index cannot describe more elements than the file has bytes (each
|
||||
// element occupies at least `offset_size` bytes). Reject a corrupt count
|
||||
// before it can drive a huge loop or overflow an offset computation.
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
#[cfg(not(feature = "std"))]
|
||||
use alloc::{string::String, vec::Vec};
|
||||
|
||||
use crate::addr::to_usize;
|
||||
use crate::btree_v1::collect_symbol_table_nodes;
|
||||
use crate::error::FormatError;
|
||||
use crate::local_heap::LocalHeap;
|
||||
@@ -54,7 +55,7 @@ pub(crate) fn v1_group_entries(
|
||||
// Parse local heap
|
||||
let heap = LocalHeap::parse(
|
||||
file_data,
|
||||
sym_table_msg.local_heap_address as usize,
|
||||
to_usize(sym_table_msg.local_heap_address)?,
|
||||
offset_size,
|
||||
length_size,
|
||||
)?;
|
||||
@@ -70,7 +71,7 @@ pub(crate) fn v1_group_entries(
|
||||
let mut entries = Vec::new();
|
||||
let mut heap_checked = false;
|
||||
for snod_addr in snod_addrs {
|
||||
let snod = SymbolTableNode::parse(file_data, snod_addr as usize, offset_size)?;
|
||||
let snod = SymbolTableNode::parse(file_data, to_usize(snod_addr)?, offset_size)?;
|
||||
for entry in &snod.entries {
|
||||
// Like libhdf5, look at the heap's free list only once a name is
|
||||
// needed: an empty group with a damaged heap still lists.
|
||||
@@ -152,7 +153,7 @@ fn for_each_v1_soft_link(
|
||||
) -> Result<(), FormatError> {
|
||||
let heap = LocalHeap::parse(
|
||||
file_data,
|
||||
sym_table_msg.local_heap_address as usize,
|
||||
to_usize(sym_table_msg.local_heap_address)?,
|
||||
offset_size,
|
||||
length_size,
|
||||
)?;
|
||||
@@ -164,7 +165,7 @@ fn for_each_v1_soft_link(
|
||||
)?;
|
||||
let mut heap_checked = false;
|
||||
for snod_addr in snod_addrs {
|
||||
let snod = SymbolTableNode::parse(file_data, snod_addr as usize, offset_size)?;
|
||||
let snod = SymbolTableNode::parse(file_data, to_usize(snod_addr)?, offset_size)?;
|
||||
for entry in &snod.entries {
|
||||
if entry.cache_type != CACHE_TYPE_SOFT_LINK {
|
||||
continue;
|
||||
@@ -242,7 +243,7 @@ pub fn resolve_path(
|
||||
// Not last — must be a group, parse its object header to get symbol table
|
||||
let obj_header = ObjectHeader::parse(
|
||||
file_data,
|
||||
entry.object_header_address as usize,
|
||||
to_usize(entry.object_header_address)?,
|
||||
offset_size,
|
||||
length_size,
|
||||
)?;
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
#[cfg(not(feature = "std"))]
|
||||
use alloc::{string::String, vec::Vec};
|
||||
|
||||
use crate::addr::to_usize;
|
||||
use crate::datatype::CharacterSet;
|
||||
use crate::error::FormatError;
|
||||
|
||||
@@ -247,7 +248,7 @@ impl LinkMessage {
|
||||
};
|
||||
|
||||
// Link name length
|
||||
let name_len = read_offset(data, pos, name_size_field_width)? as usize;
|
||||
let name_len = to_usize(read_offset(data, pos, name_size_field_width)?)?;
|
||||
pos += name_size_field_width as usize;
|
||||
|
||||
// Link name
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
#[cfg(not(feature = "std"))]
|
||||
use alloc::string::String;
|
||||
|
||||
use crate::addr::to_usize;
|
||||
use crate::error::FormatError;
|
||||
|
||||
/// Parsed HDF5 Local Heap header.
|
||||
@@ -140,15 +141,15 @@ impl LocalHeap {
|
||||
|
||||
/// Read a null-terminated string from the heap's data segment at the given byte offset.
|
||||
pub fn read_string(&self, file_data: &[u8], string_offset: u64) -> Result<String, FormatError> {
|
||||
let seg_addr = self.data_segment_address as usize;
|
||||
let seg_addr = to_usize(self.data_segment_address)?;
|
||||
let str_start =
|
||||
seg_addr
|
||||
.checked_add(string_offset as usize)
|
||||
.checked_add(to_usize(string_offset)?)
|
||||
.ok_or(FormatError::Overflow(
|
||||
"local heap seg_addr + string_offset overflow".into(),
|
||||
))?;
|
||||
let seg_end = seg_addr
|
||||
.checked_add(self.data_segment_size as usize)
|
||||
.checked_add(to_usize(self.data_segment_size)?)
|
||||
.ok_or(FormatError::Overflow(
|
||||
"local heap seg_addr + data_segment_size overflow".into(),
|
||||
))?;
|
||||
|
||||
@@ -5,6 +5,7 @@ use alloc::vec::Vec;
|
||||
|
||||
use byteorder::{ByteOrder, LittleEndian};
|
||||
|
||||
use crate::addr::to_usize;
|
||||
use crate::error::FormatError;
|
||||
use crate::message_type::MessageType;
|
||||
|
||||
@@ -264,8 +265,8 @@ impl ObjectHeader {
|
||||
// Follow continuations (v1 continuation chunks are just raw
|
||||
// messages, no signature); check_message has checked the body.
|
||||
if msg_type == MessageType::ObjectHeaderContinuation {
|
||||
let cont_offset = read_offset(body, 0, offset_size)? as usize;
|
||||
let cont_length = read_offset(body, offset_size as usize, length_size)? as usize;
|
||||
let cont_offset = to_usize(read_offset(body, 0, offset_size)?)?;
|
||||
let cont_length = to_usize(read_offset(body, offset_size as usize, length_size)?)?;
|
||||
Self::parse_v1_chunk(
|
||||
data,
|
||||
cont_offset,
|
||||
@@ -339,7 +340,7 @@ impl ObjectHeader {
|
||||
_ => unreachable!(),
|
||||
};
|
||||
ensure_len(data, pos, chunk_size_width as usize)?;
|
||||
let chunk0_size = read_offset(data, pos, chunk_size_width)? as usize;
|
||||
let chunk0_size = to_usize(read_offset(data, pos, chunk_size_width)?)?;
|
||||
pos += chunk_size_width as usize;
|
||||
// Bit 2: attribute creation order tracked → messages include creation order field
|
||||
let has_creation_order = flags & 0x04 != 0;
|
||||
@@ -472,8 +473,8 @@ impl ObjectHeader {
|
||||
let msg_type = MessageType::from_u16(msg_type_raw);
|
||||
if msg_type == MessageType::ObjectHeaderContinuation {
|
||||
// check_message has checked the body holds both fields.
|
||||
let cont_off = read_offset(body, 0, offset_size)? as usize;
|
||||
let cont_len = read_offset(body, offset_size as usize, length_size)? as usize;
|
||||
let cont_off = to_usize(read_offset(body, 0, offset_size)?)?;
|
||||
let cont_len = to_usize(read_offset(body, offset_size as usize, length_size)?)?;
|
||||
continuations.push((cont_off, cont_len));
|
||||
} else if msg_type == MessageType::Nil {
|
||||
null_count += 1;
|
||||
|
||||
@@ -203,7 +203,12 @@ fn copy_overlap(
|
||||
};
|
||||
let (src_strides, out_strides) = (strides(src_shape), strides(box_extent));
|
||||
let last = rank - 1;
|
||||
let run = ((hi[last] - lo[last]) as usize) * elem_size;
|
||||
// Byte offsets into the in-memory buffers; one that does not fit `usize`
|
||||
// (a 32-bit target) is out of both buffers, like one past their ends.
|
||||
let bytes = |elements: u64| usize::try_from(elements).ok()?.checked_mul(elem_size);
|
||||
let Some(run) = bytes(hi[last] - lo[last]) else {
|
||||
return;
|
||||
};
|
||||
|
||||
let mut idx = lo.clone();
|
||||
loop {
|
||||
@@ -213,8 +218,12 @@ fn copy_overlap(
|
||||
let out_at: u64 = (0..rank)
|
||||
.map(|d| (idx[d] - box_start[d]) * out_strides[d])
|
||||
.sum();
|
||||
let (s, o) = (src_at as usize * elem_size, out_at as usize * elem_size);
|
||||
if let (Some(from), Some(to)) = (src.get(s..s + run), out.get_mut(o..o + run)) {
|
||||
if let (Some(s), Some(o)) = (bytes(src_at), bytes(out_at))
|
||||
&& let (Some(from), Some(to)) = (
|
||||
src.get(s..s.saturating_add(run)),
|
||||
out.get_mut(o..o.saturating_add(run)),
|
||||
)
|
||||
{
|
||||
to.copy_from_slice(from);
|
||||
}
|
||||
// Advance over every dimension but the last.
|
||||
|
||||
@@ -19,6 +19,7 @@ use alloc::{vec, vec::Vec};
|
||||
|
||||
use core::ops::Range;
|
||||
|
||||
use crate::addr::to_usize;
|
||||
use crate::error::FormatError;
|
||||
|
||||
/// A selection describing which elements of a dataset to access.
|
||||
@@ -562,7 +563,7 @@ fn decode_hyperslab(r: &mut SelReader, version: u64) -> Result<SerializedSelecti
|
||||
if !matches!(enc_size, 2 | 4 | 8) {
|
||||
return Err(sel_err("unsupported hyperslab coordinate encoding size"));
|
||||
}
|
||||
let rank = r.uint(4)? as usize;
|
||||
let rank = to_usize(r.uint(4)?)?;
|
||||
// HDF5 caps dataspace rank at 32 (H5S_MAX_RANK). Reject anything else so a
|
||||
// corrupt rank can't drive a huge allocation or read loop.
|
||||
if rank == 0 || rank > 32 {
|
||||
@@ -625,11 +626,11 @@ fn decode_hyperslab(r: &mut SelReader, version: u64) -> Result<SerializedSelecti
|
||||
return Err(FormatError::UnexpectedEof {
|
||||
expected: r
|
||||
.pos
|
||||
.saturating_add(nblocks.saturating_mul(per_block) as usize),
|
||||
.saturating_add(to_usize(nblocks.saturating_mul(per_block))?),
|
||||
available: r.data.len(),
|
||||
});
|
||||
}
|
||||
let n = nblocks as usize * rank;
|
||||
let n = to_usize(nblocks)? * rank;
|
||||
let (mut starts, mut ends) = (Vec::with_capacity(n), Vec::with_capacity(n));
|
||||
for _ in 0..nblocks {
|
||||
for _ in 0..rank {
|
||||
@@ -662,7 +663,7 @@ fn blocks_union_coords(
|
||||
.filter(|&t| t <= MAX_EXPANDED_POINTS)
|
||||
.ok_or_else(|| sel_err("irregular hyperslab selection is too large to expand"))?;
|
||||
}
|
||||
let mut out = Vec::with_capacity(total as usize);
|
||||
let mut out = Vec::with_capacity(to_usize(total)?);
|
||||
for (s, e) in starts.chunks_exact(rank).zip(ends.chunks_exact(rank)) {
|
||||
let mut cur = s.to_vec();
|
||||
'block: loop {
|
||||
|
||||
@@ -23,6 +23,7 @@ use alloc::vec::Vec;
|
||||
#[cfg(feature = "std")]
|
||||
use std::borrow::Cow;
|
||||
|
||||
use crate::addr::to_usize;
|
||||
use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records};
|
||||
use crate::error::FormatError;
|
||||
use crate::fractal_heap::FractalHeapHeader;
|
||||
@@ -421,7 +422,7 @@ pub fn load_sohm_table(
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
let ext = ObjectHeader::parse(file_data, ext_addr as usize, offset_size, length_size)?;
|
||||
let ext = ObjectHeader::parse(file_data, to_usize(ext_addr)?, offset_size, length_size)?;
|
||||
let Some(msg) = ext
|
||||
.messages
|
||||
.iter()
|
||||
@@ -432,7 +433,7 @@ pub fn load_sohm_table(
|
||||
let table_msg = parse_sohm_table_message(&msg.data, offset_size)?;
|
||||
parse_sohm_table(
|
||||
file_data,
|
||||
table_msg.table_address as usize,
|
||||
to_usize(table_msg.table_address)?,
|
||||
table_msg.nindexes,
|
||||
offset_size,
|
||||
)
|
||||
@@ -505,7 +506,7 @@ pub fn resolve_sohm_message(
|
||||
|
||||
let fh_header = FractalHeapHeader::parse(
|
||||
file_data,
|
||||
index.heap_addr as usize,
|
||||
to_usize(index.heap_addr)?,
|
||||
offset_size,
|
||||
length_size,
|
||||
)?;
|
||||
@@ -587,7 +588,7 @@ pub fn resolve_shared_message_with_sohm(
|
||||
) {
|
||||
(Some(addr), _) => {
|
||||
let target_header =
|
||||
ObjectHeader::parse(file_data, addr as usize, offset_size, length_size)?;
|
||||
ObjectHeader::parse(file_data, to_usize(addr)?, offset_size, length_size)?;
|
||||
for msg in &target_header.messages {
|
||||
if msg.msg_type == target_msg_type && !is_shared(msg.flags) {
|
||||
return Ok(msg.data.clone());
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
#[cfg(not(feature = "std"))]
|
||||
use alloc::{format, string::String, vec, vec::Vec};
|
||||
|
||||
use crate::addr::to_usize;
|
||||
use crate::data_layout::{DataLayout, VdsMapping, parse_vds_mappings};
|
||||
use crate::dataspace::Dataspace;
|
||||
use crate::datatype::Datatype;
|
||||
@@ -208,7 +209,7 @@ fn load_mappings(
|
||||
return Ok(Vec::new());
|
||||
};
|
||||
let coll =
|
||||
crate::global_heap::GlobalHeapCollection::parse(file_data, addr as usize, length_size)?;
|
||||
crate::global_heap::GlobalHeapCollection::parse(file_data, to_usize(addr)?, length_size)?;
|
||||
let index = u16::try_from(*global_heap_index)
|
||||
.map_err(|_| vds_err("VDS mapping heap index out of range"))?;
|
||||
let obj = coll
|
||||
@@ -611,12 +612,12 @@ fn scatter(
|
||||
return Err(vds_err("virtual/source selection element counts differ"));
|
||||
}
|
||||
for (&v, &s) in vidx.iter().zip(sidx) {
|
||||
let (vo, so) = (v as usize * elem_size, s as usize * elem_size);
|
||||
let (vo, so) = (to_usize(v)? * elem_size, to_usize(s)? * elem_size);
|
||||
if vo + elem_size > out.len() || so + elem_size > src.len() {
|
||||
return Err(vds_err("virtual dataset selection out of bounds"));
|
||||
}
|
||||
out[vo..vo + elem_size].copy_from_slice(&src[so..so + elem_size]);
|
||||
mapped[v as usize] = true;
|
||||
mapped[to_usize(v)?] = true;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -747,7 +748,7 @@ fn selection_indices(
|
||||
return Err(vds_err("VDS selection blocks overlap"));
|
||||
}
|
||||
}
|
||||
let mut out = Vec::with_capacity(volume as usize);
|
||||
let mut out = Vec::with_capacity(to_usize(volume)?);
|
||||
for (s, e) in starts.chunks_exact(rank).zip(ends.chunks_exact(rank)) {
|
||||
let mut cur = s.to_vec();
|
||||
'block: loop {
|
||||
@@ -868,7 +869,7 @@ fn load_source_file(whole: &mut [u8]) -> Result<(), FormatError> {
|
||||
// read as before, up to its length.
|
||||
let end = sb
|
||||
.data_end(base as u64, whole.len() as u64)
|
||||
.map_or(whole.len(), |e| base + e as usize);
|
||||
.map_or(Ok(whole.len()), |e| to_usize(e).map(|e| base + e))?;
|
||||
crate::superblock_ext::apply_cache_image_in_place(&mut whole[base..end], &sb)
|
||||
}
|
||||
|
||||
@@ -921,7 +922,7 @@ fn open_source(file_data: &[u8], path: &str) -> Result<Option<OpenSource>, Forma
|
||||
Err(FormatError::PathNotFound(_)) => return Ok(None),
|
||||
Err(e) => return Err(e),
|
||||
};
|
||||
let header = crate::object_header::ObjectHeader::parse(file_data, addr as usize, os, ls)?;
|
||||
let header = crate::object_header::ObjectHeader::parse(file_data, to_usize(addr)?, os, ls)?;
|
||||
let mut src = OpenSource {
|
||||
offset_size: os,
|
||||
length_size: ls,
|
||||
|
||||
@@ -9,6 +9,7 @@ use alloc::{collections::BTreeMap, format, string::String, vec, vec::Vec};
|
||||
#[cfg(feature = "std")]
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use crate::addr::to_usize;
|
||||
use crate::error::FormatError;
|
||||
use crate::global_heap::{GlobalHeapCollection, GlobalHeapIndex};
|
||||
|
||||
@@ -55,7 +56,7 @@ pub fn parse_vl_references(
|
||||
) -> Result<Vec<VlElement>, FormatError> {
|
||||
let elem_size = 4 + offset_size as usize + 4; // length + address + index
|
||||
let total =
|
||||
(num_elements as usize)
|
||||
to_usize(num_elements)?
|
||||
.checked_mul(elem_size)
|
||||
.ok_or(FormatError::UnexpectedEof {
|
||||
expected: usize::MAX,
|
||||
@@ -68,7 +69,7 @@ pub fn parse_vl_references(
|
||||
});
|
||||
}
|
||||
|
||||
let mut elements = Vec::with_capacity(num_elements as usize);
|
||||
let mut elements = Vec::with_capacity(to_usize(num_elements)?);
|
||||
let mut pos = 0;
|
||||
|
||||
for _ in 0..num_elements {
|
||||
@@ -406,7 +407,7 @@ impl<'a> VlResolver<'a> {
|
||||
let index =
|
||||
GlobalHeapCollection::parse_index(self.file_data, offset, self.length_size)?;
|
||||
// parse_index checked that the collection lies in the file.
|
||||
let end = offset + index.collection_size as usize;
|
||||
let end = offset + to_usize(index.collection_size)?;
|
||||
self.check_overlap(offset, end)?;
|
||||
let coll = CachedCollection::new(index);
|
||||
if self.cached_bytes.saturating_add(coll.cost()) > self.budget {
|
||||
|
||||
Executable
+35
@@ -0,0 +1,35 @@
|
||||
#!/usr/bin/env bash
|
||||
# CI check: clawhdf5-format has no truncating `u64 as usize` cast on a 32-bit
|
||||
# target. HDF5 addresses and lengths are 64-bit; on wasm32 (or any 32-bit
|
||||
# target) such a cast silently wraps an address past 4 GiB onto another part
|
||||
# of the file. File values go through `addr::to_usize` (a clean error) and
|
||||
# in-memory counts through `addr::saturating_usize`.
|
||||
#
|
||||
# Lints wasm32-unknown-unknown with clippy's cast_possible_truncation and
|
||||
# fails on any u64 -> usize finding (other truncations are not checked here).
|
||||
#
|
||||
# Usage:
|
||||
# ./scripts/check-32bit-casts.sh
|
||||
#
|
||||
# Prerequisites:
|
||||
# rustup target add wasm32-unknown-unknown
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
TARGET="wasm32-unknown-unknown"
|
||||
echo "==> Checking for truncating u64 -> usize casts in clawhdf5-format ($TARGET)"
|
||||
|
||||
out=$(cargo clippy -p clawhdf5-format --target "$TARGET" \
|
||||
--features plugin-filters --message-format short \
|
||||
-- -A clippy::all -W clippy::cast_possible_truncation 2>&1) || {
|
||||
echo "$out"
|
||||
echo "==> clippy failed" >&2
|
||||
exit 1
|
||||
}
|
||||
found=$(grep -F 'casting `u64` to `usize`' <<<"$out" || true)
|
||||
if [ -n "$found" ]; then
|
||||
echo "$found"
|
||||
echo "==> use addr::to_usize (file values) or addr::saturating_usize (in-memory counts)" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "==> no truncating u64 -> usize casts"
|
||||
@@ -147,6 +147,8 @@ run_step "wasm32 clippy (clawhdf5-wasm)" cargo clippy \
|
||||
--target wasm32-unknown-unknown \
|
||||
--all-targets \
|
||||
-- -D warnings
|
||||
# A 64-bit file address must not wrap on a 32-bit target.
|
||||
run_step "check-32bit-casts.sh" "$SCRIPT_DIR/check-32bit-casts.sh"
|
||||
|
||||
# The built wasm package, run under Node against h5py/netCDF4-written files,
|
||||
# and the viewer page in headless Chromium when one is found.
|
||||
|
||||
Reference in New Issue
Block a user