Range reads M0/M1 (indexed lookups, Storage trait), ZFP, in-place editing #17

Merged
osobh merged 52 commits from feat/p3-range-zfp-edit into main 2026-09-26 20:42:22 +00:00
27 changed files with 236 additions and 99 deletions
Showing only changes of commit b41583113a - Show all commits
+22
View File
@@ -23,6 +23,19 @@ pub fn to_usize(value: u64) -> Result<usize, FormatError> {
usize::try_from(value).map_err(|_| too_large(value))
}
/// A count or offset into an in-memory buffer (a codec's progress counter,
/// a size the writer computed from data it holds) as a `usize`, saturating
/// at `usize::MAX` instead of truncating.
///
/// For values that are bounded by the length of something in memory, so
/// always fit; if one ever did not, a saturated index fails its bounds check
/// or allocation instead of silently addressing the wrong bytes. A value
/// read from the file uses [`to_usize`].
#[inline]
pub fn saturating_usize(value: u64) -> usize {
usize::try_from(value).unwrap_or(usize::MAX)
}
#[cold]
#[inline(never)]
fn too_large(value: u64) -> FormatError {
@@ -42,6 +55,15 @@ mod tests {
assert_eq!(to_usize(usize::MAX as u64), Ok(usize::MAX));
}
#[test]
fn saturating_conversion_never_wraps() {
assert_eq!(saturating_usize(0), 0);
assert_eq!(saturating_usize(0x1234), 0x1234);
assert_eq!(saturating_usize(usize::MAX as u64), usize::MAX);
// Past usize::MAX (32-bit targets) or at u64::MAX: saturates.
assert_eq!(saturating_usize(u64::MAX), usize::MAX);
}
#[test]
fn values_past_usize_max_are_an_error_not_truncated() {
// Only reachable where usize is narrower than u64; on a 64-bit host
+7 -1
View File
@@ -3,6 +3,7 @@
#[cfg(not(feature = "std"))]
use alloc::vec::Vec;
use crate::addr::to_usize;
use crate::error::FormatError;
/// A parsed B-tree v1 node.
@@ -164,7 +165,12 @@ fn collect_symbol_table_nodes_inner(
return Err(FormatError::NestingDepthExceeded);
}
let node = BTreeV1Node::parse(file_data, btree_address as usize, offset_size, length_size)?;
let node = BTreeV1Node::parse(
file_data,
to_usize(btree_address)?,
offset_size,
length_size,
)?;
if node.node_type != 0 {
return Err(FormatError::InvalidBTreeNodeType(node.node_type));
+5 -2
View File
@@ -6,6 +6,7 @@
//! libhdf5 uses (`H5B2__hdr_init`) and the reader decodes pointers with, so
//! the pointer widths the writer encodes are the ones every reader expects.
use crate::addr::saturating_usize;
#[cfg(not(feature = "std"))]
use alloc::{format, vec, vec::Vec};
@@ -105,7 +106,9 @@ pub(crate) fn build_btree_v2(
first_node: addr + hdr_len as u64,
nodes: Vec::new(),
};
let root = (n > 0).then(|| w.node(depth, 0, n as usize)).transpose()?;
let root = (n > 0)
.then(|| w.node(depth, 0, saturating_usize(n)))
.transpose()?;
let mut out = Vec::with_capacity(hdr_len + w.nodes.len() * p.node_size as usize);
out.extend_from_slice(b"BTHD");
@@ -201,7 +204,7 @@ impl TreeWriter<'_> {
"cannot spread {n} B-tree v2 records over {k} children at depth {depth}"
)));
}
let k = k as usize;
let k = saturating_usize(k);
let in_children = n - (k - 1);
let (base, extra) = (in_children / k, in_children % k);
+10 -1
View File
@@ -18,6 +18,7 @@ use alloc::collections::BTreeMap;
#[cfg(feature = "std")]
use std::collections::HashMap;
use crate::addr::to_usize;
use crate::chunk_cache::ChunkCoord;
use crate::chunked_read::ChunkInfo;
@@ -167,7 +168,15 @@ impl ChunkLayout {
for (_coord, ci) in index.iter() {
let coord: ChunkCoord = ci.offsets.iter().take(rank).copied().collect();
let chunk_offsets: Vec<usize> = coord.iter().map(|&o| o as usize).collect();
// `ds_dims` are `usize`: a chunk at an offset past `usize::MAX`
// (only on a 32-bit target) lies outside the dataset.
let Ok(chunk_offsets) = coord
.iter()
.map(|&o| to_usize(o))
.collect::<Result<Vec<usize>, _>>()
else {
continue;
};
let copies = if rank == 0 {
// Scalar dataset — single copy
+32 -14
View File
@@ -6,6 +6,7 @@ extern crate alloc;
#[cfg(not(feature = "std"))]
use alloc::{format, vec, vec::Vec};
use crate::addr::to_usize;
#[cfg(feature = "std")]
use crate::chunk_cache::{CacheAlignedBuffer, ChunkCache};
use crate::data_layout::DataLayout;
@@ -265,7 +266,7 @@ fn fill_from_chunks(
)));
}
let offsets = &c.offsets[..rank];
let c_addr = c.address as usize;
let c_addr = to_usize(c.address)?;
let size = c.chunk_size as usize;
ensure_len(file_data, c_addr, size)?;
let raw = &file_data[c_addr..c_addr + size];
@@ -825,7 +826,7 @@ fn parse_chunk_node(
return Err(FormatError::NestingDepthExceeded);
}
let offset = btree_address as usize;
let offset = to_usize(btree_address)?;
let os = offset_size as usize;
// Parse B-tree v1 header
@@ -945,7 +946,8 @@ pub fn generate_implicit_chunks(
}
let total_chunks: u64 = num_chunks_per_dim.iter().product();
let mut chunks = Vec::with_capacity(total_chunks as usize);
// A capacity hint only (a count past `usize::MAX` could not be pushed).
let mut chunks = Vec::with_capacity(usize::try_from(total_chunks).unwrap_or(0));
for linear_idx in 0..total_chunks {
let mut offsets = vec![0u64; rank];
let mut remaining = linear_idx;
@@ -993,7 +995,7 @@ fn read_btree_v2_chunks(
use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records};
let bad = |what: &str| FormatError::ChunkedReadError(format!("B-tree v2 chunk index: {what}"));
let header = BTreeV2Header::parse(file_data, addr as usize, offset_size, length_size)?;
let header = BTreeV2Header::parse(file_data, to_usize(addr)?, offset_size, length_size)?;
let rank = chunk_dims.len();
let os = offset_size as usize;
let record_size = header.record_size as usize;
@@ -1122,7 +1124,11 @@ pub fn list_chunks(
// Both v3 and v4 include element size as last dim (rank+1)
let (rank, chunk_dims) = chunk_geometry(chunk_dimensions, version, dataspace, elem_size)?;
let ds_dims: Vec<usize> = dataspace.dimensions.iter().map(|&d| d as usize).collect();
let ds_dims: Vec<usize> = dataspace
.dimensions
.iter()
.map(|&d| to_usize(d))
.collect::<Result<_, _>>()?;
// Collect chunks based on version and index type
let mut chunks = match (version, chunk_index_type) {
@@ -1158,7 +1164,7 @@ pub fn list_chunks(
// Fixed Array — use spatial chunk dims only
let spatial_chunk_dims: &[u32] = &chunk_dimensions[..rank];
let header =
FixedArrayHeader::parse(file_data, addr as usize, offset_size, length_size)?;
FixedArrayHeader::parse(file_data, to_usize(addr)?, offset_size, length_size)?;
read_fixed_array_chunks(
file_data,
&header,
@@ -1174,7 +1180,7 @@ pub fn list_chunks(
// Extensible Array — use spatial chunk dims only
let spatial_chunk_dims: &[u32] = &chunk_dimensions[..rank];
let header =
ExtensibleArrayHeader::parse(file_data, addr as usize, offset_size, length_size)?;
ExtensibleArrayHeader::parse(file_data, to_usize(addr)?, offset_size, length_size)?;
read_extensible_array_chunks(
file_data,
&header,
@@ -1349,7 +1355,11 @@ pub(crate) fn read_chunked_full<O>(
// dimension the total is 0 even if other dimensions are huge.
return Ok(output);
}
let ds_dims: Vec<usize> = dataspace.dimensions.iter().map(|&d| d as usize).collect();
let ds_dims: Vec<usize> = dataspace
.dimensions
.iter()
.map(|&d| to_usize(d))
.collect::<Result<_, _>>()?;
let placer = ChunkPlacer::new(&chunk_dims, &ds_dims, elem_size);
let chunk_total_bytes = checked_chunk_byte_len(&chunk_dims, elem_size)?;
// Chunks are cached only when the whole dataset fits: pushing a larger
@@ -1603,7 +1613,11 @@ pub fn read_chunked_data_sweep(
check_chunk_element_size(layout, datatype, offset_size)?;
let elem_size = datatype.type_size() as usize;
let (rank, chunk_dims) = chunk_geometry(chunk_dimensions, version, dataspace, elem_size)?;
let ds_dims: Vec<usize> = dataspace.dimensions.iter().map(|&d| d as usize).collect();
let ds_dims: Vec<usize> = dataspace
.dimensions
.iter()
.map(|&d| to_usize(d))
.collect::<Result<_, _>>()?;
// The per-file cache is shared across datasets (and threads); every
// lookup is keyed by this dataset's chunk-index address, so another
@@ -1659,7 +1673,7 @@ pub fn read_chunked_data_sweep(
cached
} else {
// Decompress from file
let c_addr = chunk_info.address as usize;
let c_addr = to_usize(chunk_info.address)?;
let size = chunk_info.chunk_size as usize;
ensure_len(file_data, c_addr, size)?;
let raw_chunk = &file_data[c_addr..c_addr + size];
@@ -1682,8 +1696,8 @@ pub fn read_chunked_data_sweep(
.offsets
.iter()
.take(rank)
.map(|&o| o as usize)
.collect();
.map(|&o| to_usize(o))
.collect::<Result<_, _>>()?;
if rank == 0 {
let copy_len = decompressed.len().min(output.len());
@@ -1743,7 +1757,11 @@ pub fn read_chunked_data_indexed(
check_chunk_element_size(layout, datatype, offset_size)?;
let elem_size = datatype.type_size() as usize;
let (rank, chunk_dims) = chunk_geometry(chunk_dimensions, version, dataspace, elem_size)?;
let ds_dims: Vec<usize> = dataspace.dimensions.iter().map(|&d| d as usize).collect();
let ds_dims: Vec<usize> = dataspace
.dimensions
.iter()
.map(|&d| to_usize(d))
.collect::<Result<_, _>>()?;
// Chunk index and assembly plan for this dataset, built on first access
// and kept per dataset (keyed by chunk-index address) in the shared cache.
@@ -1776,7 +1794,7 @@ pub fn read_chunked_data_indexed(
if let Some(cached) = cache.get_decompressed_in(addr, coord) {
chunk_buffers.push(cached);
} else {
let c_addr = *file_offset as usize;
let c_addr = to_usize(*file_offset)?;
let size = *file_size as usize;
ensure_len(file_data, c_addr, size)?;
let raw_chunk = &file_data[c_addr..c_addr + size];
+12 -11
View File
@@ -3,6 +3,7 @@
#[cfg(not(feature = "std"))]
extern crate alloc;
use crate::addr::saturating_usize;
#[cfg(not(feature = "std"))]
use alloc::{format, vec, vec::Vec};
@@ -414,18 +415,18 @@ pub fn split_into_chunks(
// Dataset strides (row-major)
let mut ds_strides = vec![1usize; rank];
for i in (0..rank.saturating_sub(1)).rev() {
ds_strides[i] = ds_strides[i + 1] * shape[i + 1] as usize;
ds_strides[i] = ds_strides[i + 1] * saturating_usize(shape[i + 1]);
}
// Chunk strides
let mut chunk_strides = vec![1usize; rank];
for i in (0..rank.saturating_sub(1)).rev() {
chunk_strides[i] = chunk_strides[i + 1] * chunk_dims[i + 1] as usize;
chunk_strides[i] = chunk_strides[i + 1] * saturating_usize(chunk_dims[i + 1]);
}
let chunk_total_elements: usize = chunk_dims.iter().map(|&d| d as usize).product();
let chunk_total_elements: usize = chunk_dims.iter().map(|&d| saturating_usize(d)).product();
let mut result = Vec::with_capacity(total_chunks as usize);
let mut result = Vec::with_capacity(saturating_usize(total_chunks));
for linear_idx in 0..total_chunks {
// Convert linear index to chunk grid coordinates
@@ -453,8 +454,8 @@ pub fn split_into_chunks(
let coord_in_chunk = remaining_idx / chunk_strides[d];
remaining_idx %= chunk_strides[d];
let global_coord = offsets[d] as usize + coord_in_chunk;
if global_coord >= shape[d] as usize {
let global_coord = saturating_usize(offsets[d]) + coord_in_chunk;
if global_coord >= saturating_usize(shape[d]) {
out_of_bounds = true;
break;
}
@@ -1036,7 +1037,7 @@ impl ChunkIndexPlan {
Ok(Self::SingleChunk)
} else {
let grid = ChunkGrid::fixed_array(shape, Some(max), chunk_dims)?;
Ok(Self::FixedArray(grid, nslots as usize))
Ok(Self::FixedArray(grid, saturating_usize(nslots)))
}
}
1 => Ok(Self::ExtensibleArray(ChunkGrid::extensible_array(
@@ -1251,7 +1252,7 @@ pub fn write_selection_to_buffer(
let rank = dims.len();
let mut ds_strides = vec![1usize; rank];
for i in (0..rank.saturating_sub(1)).rev() {
ds_strides[i] = ds_strides[i + 1] * dims[i + 1] as usize;
ds_strides[i] = ds_strides[i + 1] * saturating_usize(dims[i + 1]);
}
let mut src_offset = 0usize;
@@ -1301,7 +1302,7 @@ pub fn write_selection_to_buffer(
buffer,
new_data,
src_offset,
current_ds_offset + coord as usize * ds_strides[d],
current_ds_offset + saturating_usize(coord) * ds_strides[d],
);
}
}
@@ -1328,14 +1329,14 @@ pub fn write_selection_to_buffer(
let rank = dims.len();
let mut ds_strides = vec![1usize; rank];
for i in (0..rank.saturating_sub(1)).rev() {
ds_strides[i] = ds_strides[i + 1] * dims[i + 1] as usize;
ds_strides[i] = ds_strides[i + 1] * saturating_usize(dims[i + 1]);
}
for (pi, pt) in pts.iter().enumerate() {
let flat: usize = pt
.iter()
.zip(ds_strides.iter())
.map(|(&p, &s)| p as usize * s)
.map(|(&p, &s)| saturating_usize(p) * s)
.sum();
let dst = flat * elem_size;
let src = pi * elem_size;
+3 -2
View File
@@ -6,6 +6,7 @@ use alloc::{format, string::String, vec::Vec};
#[cfg(feature = "std")]
use std::string::String;
use crate::addr::to_usize;
use crate::error::FormatError;
/// A single VDS (Virtual Dataset) source mapping.
@@ -207,7 +208,7 @@ pub fn parse_vds_mappings(
"VDS mapping shares a name with a later entry".into(),
));
}
Ok(idx as usize)
to_usize(idx)
};
let source_file = if flags & VDS_SOURCE_SAME_FILE != 0 {
@@ -320,7 +321,7 @@ impl DataLayout {
{
let coll = crate::global_heap::GlobalHeapCollection::parse(
file_data,
addr as usize,
to_usize(addr)?,
length_size,
)?;
let obj = coll.get_object(*global_heap_index as u16).ok_or(
+9 -8
View File
@@ -6,6 +6,7 @@ use alloc::{collections::BTreeMap, format, string::String, vec, vec::Vec};
#[cfg(feature = "std")]
use std::collections::BTreeMap;
use crate::addr::to_usize;
#[cfg(feature = "std")]
use crate::chunk_cache::ChunkCache;
use crate::chunked_read::read_chunked_data;
@@ -117,7 +118,7 @@ pub fn read_raw_data_zerocopy<'a>(
dataspace: &Dataspace,
datatype: &Datatype,
) -> Result<Option<&'a [u8]>, FormatError> {
let num_elements = dataspace.num_elements() as usize;
let num_elements = to_usize(dataspace.num_elements())?;
let elem_size = datatype.type_size() as usize;
let expected_size = num_elements.checked_mul(elem_size).ok_or_else(|| {
FormatError::Overflow(format!(
@@ -128,7 +129,7 @@ pub fn read_raw_data_zerocopy<'a>(
match layout {
DataLayout::Contiguous { address, size } => {
let addr = address.ok_or(FormatError::NoDataAllocated)?;
let addr = addr as usize;
let addr = to_usize(addr)?;
let sz = contiguous_read_len(*size, expected_size)?;
ensure_len(file_data, addr, sz)?;
Ok(Some(&file_data[addr..addr + sz]))
@@ -219,7 +220,7 @@ fn read_raw_data_full_impl(
length_size: u8,
resolver: Option<&VdsSourceResolver>,
) -> Result<Vec<u8>, FormatError> {
let num_elements = dataspace.num_elements() as usize;
let num_elements = to_usize(dataspace.num_elements())?;
let elem_size = datatype.type_size() as usize;
let expected_size = num_elements.checked_mul(elem_size).ok_or_else(|| {
FormatError::Overflow(format!(
@@ -239,7 +240,7 @@ fn read_raw_data_full_impl(
}
DataLayout::Contiguous { address, size } => {
let addr = address.ok_or(FormatError::NoDataAllocated)?;
let addr = addr as usize;
let addr = to_usize(addr)?;
let sz = contiguous_read_len(*size, expected_size)?;
ensure_len(file_data, addr, sz)?;
let mut out = crate::bulk_alloc::vec_for_bulk(sz);
@@ -582,7 +583,7 @@ pub fn extract_selection_from_buffer(
let rank = dims.len();
let mut ds_strides = vec![1usize; rank];
for i in (0..rank.saturating_sub(1)).rev() {
ds_strides[i] = ds_strides[i + 1] * dims[i + 1] as usize;
ds_strides[i] = ds_strides[i + 1] * to_usize(dims[i + 1])?;
}
let mut output = Vec::with_capacity(pts.len() * elem_size);
@@ -590,8 +591,8 @@ pub fn extract_selection_from_buffer(
let flat: usize = pt
.iter()
.zip(ds_strides.iter())
.map(|(&p, &s)| p as usize * s)
.sum();
.map(|(&p, &s)| Ok(to_usize(p)? * s))
.sum::<Result<usize, FormatError>>()?;
let src = flat * elem_size;
if src + elem_size <= full_data.len() {
output.extend_from_slice(&full_data[src..src + elem_size]);
@@ -1341,7 +1342,7 @@ pub fn read_compound_fields(
let mut fields = Vec::with_capacity(members.len());
for m in members {
let field_size = m.datatype.type_size() as usize;
let offset = m.byte_offset as usize;
let offset = to_usize(m.byte_offset)?;
if offset
.checked_add(field_size)
.is_none_or(|end| end > elem_size)
+2 -1
View File
@@ -3,6 +3,7 @@
#[cfg(not(feature = "std"))]
extern crate alloc;
use crate::addr::saturating_usize;
#[cfg(not(feature = "std"))]
use alloc::{vec, vec::Vec};
@@ -247,7 +248,7 @@ pub fn build_extensible_array_at(
// Header (EAHD). The six statistics are, in order: super blocks, their
// bytes, data blocks, their bytes, max index set, elements realised.
let mut out = Vec::with_capacity((cursor - ea_base_address) as usize);
let mut out = Vec::with_capacity(saturating_usize(cursor - ea_base_address));
out.extend_from_slice(b"EAHD");
out.push(0); // version
out.push(client_id);
@@ -9,6 +9,7 @@ extern crate alloc;
#[cfg(not(feature = "std"))]
use alloc::{format, vec, vec::Vec};
use crate::addr::to_usize;
use crate::chunk_grid::ChunkGrid;
use crate::chunked_read::ChunkInfo;
use crate::error::FormatError;
@@ -451,7 +452,7 @@ pub fn read_extensible_array_chunks(
// Parse index block (EAIB): signature(4) + version(1) + client_id(1)
// + header address(offset_size), then the inline elements, then the
// direct data block addresses, then the super block addresses.
let ib_offset = header.index_block_address as usize;
let ib_offset = to_usize(header.index_block_address)?;
let ib_header_size = 4 + 1 + 1 + os;
ensure_len(file_data, ib_offset, ib_header_size)?;
@@ -463,7 +464,7 @@ pub fn read_extensible_array_chunks(
let mut pos = ib_offset + ib_header_size;
let mut chunks = Vec::new();
let total_elements = header.num_elements as usize;
let total_elements = to_usize(header.num_elements)?;
let dmin = header.min_dblk_nelmts as usize;
if dmin == 0 || !dmin.is_power_of_two() {
@@ -563,7 +564,7 @@ pub fn read_extensible_array_chunks(
}
chunks.extend(read_data_block_elements(
file_data,
addr as usize,
to_usize(addr)?,
dblk_nelmts,
header,
offset_size,
@@ -592,7 +593,7 @@ pub fn read_extensible_array_chunks(
if !is_undefined_addr(sb_addr, offset_size) {
chunks.extend(read_super_block(
file_data,
sb_addr as usize,
to_usize(sb_addr)?,
ndblks,
dblk_nelmts,
header,
@@ -676,7 +677,7 @@ fn read_super_block(
if !is_undefined_addr(addr, offset_size) {
chunks.extend(read_data_block_elements(
file_data,
addr as usize,
to_usize(addr)?,
dblk_nelmts,
header,
offset_size,
+11 -9
View File
@@ -3,6 +3,7 @@
//! Produces valid HDF5 files with v3 superblock, v2 object headers,
//! link messages, contiguous datasets, inline and dense attributes.
use crate::addr::saturating_usize;
#[cfg(not(feature = "std"))]
use alloc::{format, vec, vec::Vec};
@@ -336,7 +337,7 @@ pub(crate) fn build_single_block_fractal_heap(
// An object must fit one direct block: the writer has no huge-object
// path, and libhdf5 cannot read an object that overruns its block.
let max_managed = max_direct_block_size as usize - dblock_header_size;
let max_managed = saturating_usize(max_direct_block_size) - dblock_header_size;
if let Some(big) = serialized.iter().find(|s| s.len() > max_managed) {
return Err(FormatError::SerializationError(format!(
"a {}-byte message cannot go in dense storage: a fractal heap \
@@ -392,7 +393,7 @@ pub(crate) fn build_single_block_fractal_heap(
let dblock_addr = frhp_addr + frhp_size as u64;
let btree_addr = dblock_addr + starting_block_size;
let data_space = starting_block_size as usize - dblock_header_size;
let data_space = saturating_usize(starting_block_size) - dblock_header_size;
let free_space = data_space - total_data_size;
// Build fractal heap header
@@ -428,7 +429,7 @@ pub(crate) fn build_single_block_fractal_heap(
debug_assert_eq!(frhp.len(), frhp_size);
// Build direct block: header (with checksum) + data + padding
let mut dblock = Vec::with_capacity(starting_block_size as usize);
let mut dblock = Vec::with_capacity(saturating_usize(starting_block_size));
dblock.extend_from_slice(b"FHDB");
dblock.push(0); // version
write_offset(&mut dblock, frhp_addr, OFFSET_SIZE);
@@ -446,12 +447,12 @@ pub(crate) fn build_single_block_fractal_heap(
}
// Pad to full block size
dblock.resize(starting_block_size as usize, 0);
dblock.resize(saturating_usize(starting_block_size), 0);
// Checksum: computed over entire block with checksum field zeroed
let dblock_checksum = crate::checksum::jenkins_lookup3(&dblock);
dblock[cksum_pos..cksum_pos + 4].copy_from_slice(&dblock_checksum.to_le_bytes());
debug_assert_eq!(dblock.len(), starting_block_size as usize);
debug_assert_eq!(dblock.len(), saturating_usize(starting_block_size));
// Build heap IDs
let heap_ids: Vec<Vec<u8>> = obj_offsets
@@ -706,7 +707,7 @@ impl HeapIndirectBlock {
let cksum_pos = out.len();
out.extend_from_slice(&[0u8; 4]); // checksum placeholder
out.extend_from_slice(&b.data);
out.resize(d + b.size as usize, 0);
out.resize(d + saturating_usize(b.size), 0);
let cksum = crate::checksum::jenkins_lookup3(&out[d..]);
out[cksum_pos..cksum_pos + 4].copy_from_slice(&cksum.to_le_bytes());
child += b.size;
@@ -740,7 +741,7 @@ impl HeapPacker<'_> {
nrows: Option<usize>,
) -> Result<HeapIndirectBlock, FormatError> {
let geom = self.geom;
let width = geom.width as usize;
let width = saturating_usize(geom.width);
let mut slots = Vec::new();
let mut off = heap_offset;
let mut row = 0usize;
@@ -763,7 +764,8 @@ impl HeapPacker<'_> {
// A child whose biggest direct block cannot hold the
// next object is skipped whole, not walked.
let biggest = geom.row_size(child_rows.min(geom.max_direct_rows()) - 1);
if self.objects[self.next].len() > (biggest as usize - geom.dblock_header_size)
if self.objects[self.next].len()
> (saturating_usize(biggest) - geom.dblock_header_size)
{
slots.push(HeapSlot::Empty);
off += size;
@@ -794,7 +796,7 @@ impl HeapPacker<'_> {
/// objects as fit; leave it unallocated if not even the next one does.
fn fill_direct(&mut self, heap_offset: u64, size: u64) -> HeapSlot {
let header = self.geom.dblock_header_size;
let capacity = size as usize - header;
let capacity = saturating_usize(size) - header;
let mut data = Vec::new();
while let Some(obj) = self.objects.get(self.next) {
if data.len() + obj.len() > capacity {
+7 -2
View File
@@ -12,6 +12,7 @@
#[cfg(not(feature = "std"))]
use alloc::{format, vec, vec::Vec};
use crate::addr::to_usize;
use crate::chunked_read::{alloc_output, checked_byte_len, list_chunks};
use crate::data_layout::DataLayout;
use crate::dataspace::Dataspace;
@@ -256,7 +257,11 @@ pub fn apply_to_unallocated_chunks(
length_size,
)?;
let rank = chunk_dims.len();
let ds_dims: Vec<usize> = dataspace.dimensions.iter().map(|&d| d as usize).collect();
let ds_dims: Vec<usize> = dataspace
.dimensions
.iter()
.map(|&d| to_usize(d))
.collect::<Result<_, _>>()?;
if rank == 0 || ds_dims.len() != rank || chunk_dims.contains(&0) {
return Ok(());
}
@@ -288,7 +293,7 @@ pub fn apply_to_unallocated_chunks(
let mut cell = 0usize;
let mut in_range = true;
for d in 0..rank {
let coord = chunk.offsets[d] as usize / chunk_dims[d];
let coord = to_usize(chunk.offsets[d])? / chunk_dims[d];
if coord >= grid[d] {
in_range = false;
break;
+13 -3
View File
@@ -3,6 +3,8 @@
#[cfg(not(feature = "std"))]
extern crate alloc;
#[cfg(feature = "deflate")]
use crate::addr::saturating_usize;
#[cfg(not(feature = "std"))]
use alloc::{boxed::Box, format, vec, vec::Vec};
@@ -1114,7 +1116,11 @@ fn inflate_bounded_into(
loop {
let (in_before, out_before) = (inflater.total_in(), inflater.total_out());
let status = inflater
.decompress_vec(&data[in_before as usize..], out, FlushDecompress::Finish)
.decompress_vec(
&data[saturating_usize(in_before)..],
out,
FlushDecompress::Finish,
)
.map_err(|e| format!("deflate: {e}"))?;
if out.len() > limit {
return Err("deflate: output exceeds size limit".into());
@@ -1132,7 +1138,7 @@ fn inflate_bounded_into(
}
Status::Ok | Status::BufError => {
// Room left, so the decoder stopped for want of input.
if inflater.total_in() as usize >= data.len()
if saturating_usize(inflater.total_in()) >= data.len()
|| (inflater.total_in(), inflater.total_out()) == (in_before, out_before)
{
return Err("deflate: truncated stream".into());
@@ -1232,7 +1238,11 @@ pub(crate) fn deflate_bounded(data: &[u8], level: u32) -> Result<Vec<u8>, String
loop {
let (in_before, out_before) = (deflater.total_in(), deflater.total_out());
let status = deflater
.compress_vec(&data[in_before as usize..], &mut out, FlushCompress::Finish)
.compress_vec(
&data[saturating_usize(in_before)..],
&mut out,
FlushCompress::Finish,
)
.map_err(|e| format!("deflate: {e}"))?;
match status {
Status::StreamEnd => return Ok(out),
+2 -1
View File
@@ -46,6 +46,7 @@
//! variable-length blocks, dictionaries, lazy chunks, user-defined codecs
//! and registered filters (e.g. bytedelta), sparse frames.
use crate::addr::saturating_usize;
use crate::error::FormatError;
use crate::filter_registry::FilterContext;
use crate::filters_bitshuffle::bitunshuffle_block;
@@ -546,7 +547,7 @@ fn parse_frame(buf: &[u8], limit: usize) -> Result<Frame<'_>, FormatError> {
return Err(err("negative size in frame header"));
}
let header_len = header_len as usize;
let buf = &buf[..frame_len as usize];
let buf = &buf[..saturating_usize(frame_len)];
let cbytes = usize::try_from(cbytes).map_err(|_| err("bad compressed size"))?;
let data_end = header_len
.checked_add(cbytes)
+4 -3
View File
@@ -4,6 +4,7 @@
//! the compression level). Decoded with the `bzip2` crate's default backend,
//! `libbz2-rs-sys`, a pure-Rust port of libbzip2.
use crate::addr::saturating_usize;
use crate::error::FormatError;
use crate::filter_registry::FilterContext;
@@ -28,7 +29,7 @@ pub(crate) fn bzip2_decode(input: &[u8], ctx: &FilterContext<'_>) -> Result<Vec<
loop {
let (in_before, out_before) = (dec.total_in(), dec.total_out());
let status = dec
.decompress_vec(&input[in_before as usize..], &mut out)
.decompress_vec(&input[saturating_usize(in_before)..], &mut out)
.map_err(|e| err(&e.to_string()))?;
if out.len() > limit {
return Err(err("output exceeds the chunk size"));
@@ -43,7 +44,7 @@ pub(crate) fn bzip2_decode(input: &[u8], ctx: &FilterContext<'_>) -> Result<Vec<
.max(1);
out.try_reserve_exact(grow)
.map_err(|_| err("cannot allocate the output buffer"))?;
} else if dec.total_in() as usize >= input.len()
} else if saturating_usize(dec.total_in()) >= input.len()
|| (dec.total_in(), dec.total_out()) == (in_before, out_before)
{
return Err(err("truncated stream"));
@@ -61,7 +62,7 @@ pub(crate) fn bzip2_encode(input: &[u8], ctx: &FilterContext<'_>) -> Result<Vec<
// bzip2's worst case is about 1% + 600 bytes over the input.
let mut out = Vec::with_capacity(input.len() + input.len() / 100 + 600);
loop {
let consumed = enc.total_in() as usize;
let consumed = saturating_usize(enc.total_in());
let status = enc
.compress_vec(&input[consumed..], &mut out, Action::Finish)
.map_err(|e| cerr(e.to_string()))?;
+3 -2
View File
@@ -6,6 +6,7 @@ extern crate alloc;
#[cfg(not(feature = "std"))]
use alloc::{format, vec, vec::Vec};
use crate::addr::to_usize;
use crate::chunk_grid::ChunkGrid;
use crate::chunked_read::ChunkInfo;
use crate::error::FormatError;
@@ -158,7 +159,7 @@ pub fn read_fixed_array_chunks(
offset_size: u8,
_length_size: u8,
) -> Result<Vec<ChunkInfo>, FormatError> {
let db_offset = header.data_block_address as usize;
let db_offset = to_usize(header.data_block_address)?;
// Parse data block header: FADB(4) + version(1) + client_id(1) + header_address(offset_size)
let db_header_size = 4 + 1 + 1 + offset_size as usize;
@@ -174,7 +175,7 @@ pub fn read_fixed_array_chunks(
// Elements start immediately after the data block prefix.
let elements_start = db_offset + db_header_size;
let num_elements = header.num_elements as usize;
let num_elements = to_usize(header.num_elements)?;
// A chunk index cannot describe more elements than the file has bytes (each
// element occupies at least `offset_size` bytes). Reject a corrupt count
// before it can drive a huge loop or overflow an offset computation.
+6 -5
View File
@@ -3,6 +3,7 @@
#[cfg(not(feature = "std"))]
use alloc::{string::String, vec::Vec};
use crate::addr::to_usize;
use crate::btree_v1::collect_symbol_table_nodes;
use crate::error::FormatError;
use crate::local_heap::LocalHeap;
@@ -54,7 +55,7 @@ pub(crate) fn v1_group_entries(
// Parse local heap
let heap = LocalHeap::parse(
file_data,
sym_table_msg.local_heap_address as usize,
to_usize(sym_table_msg.local_heap_address)?,
offset_size,
length_size,
)?;
@@ -70,7 +71,7 @@ pub(crate) fn v1_group_entries(
let mut entries = Vec::new();
let mut heap_checked = false;
for snod_addr in snod_addrs {
let snod = SymbolTableNode::parse(file_data, snod_addr as usize, offset_size)?;
let snod = SymbolTableNode::parse(file_data, to_usize(snod_addr)?, offset_size)?;
for entry in &snod.entries {
// Like libhdf5, look at the heap's free list only once a name is
// needed: an empty group with a damaged heap still lists.
@@ -152,7 +153,7 @@ fn for_each_v1_soft_link(
) -> Result<(), FormatError> {
let heap = LocalHeap::parse(
file_data,
sym_table_msg.local_heap_address as usize,
to_usize(sym_table_msg.local_heap_address)?,
offset_size,
length_size,
)?;
@@ -164,7 +165,7 @@ fn for_each_v1_soft_link(
)?;
let mut heap_checked = false;
for snod_addr in snod_addrs {
let snod = SymbolTableNode::parse(file_data, snod_addr as usize, offset_size)?;
let snod = SymbolTableNode::parse(file_data, to_usize(snod_addr)?, offset_size)?;
for entry in &snod.entries {
if entry.cache_type != CACHE_TYPE_SOFT_LINK {
continue;
@@ -242,7 +243,7 @@ pub fn resolve_path(
// Not last — must be a group, parse its object header to get symbol table
let obj_header = ObjectHeader::parse(
file_data,
entry.object_header_address as usize,
to_usize(entry.object_header_address)?,
offset_size,
length_size,
)?;
+2 -1
View File
@@ -3,6 +3,7 @@
#[cfg(not(feature = "std"))]
use alloc::{string::String, vec::Vec};
use crate::addr::to_usize;
use crate::datatype::CharacterSet;
use crate::error::FormatError;
@@ -247,7 +248,7 @@ impl LinkMessage {
};
// Link name length
let name_len = read_offset(data, pos, name_size_field_width)? as usize;
let name_len = to_usize(read_offset(data, pos, name_size_field_width)?)?;
pos += name_size_field_width as usize;
// Link name
+4 -3
View File
@@ -3,6 +3,7 @@
#[cfg(not(feature = "std"))]
use alloc::string::String;
use crate::addr::to_usize;
use crate::error::FormatError;
/// Parsed HDF5 Local Heap header.
@@ -140,15 +141,15 @@ impl LocalHeap {
/// Read a null-terminated string from the heap's data segment at the given byte offset.
pub fn read_string(&self, file_data: &[u8], string_offset: u64) -> Result<String, FormatError> {
let seg_addr = self.data_segment_address as usize;
let seg_addr = to_usize(self.data_segment_address)?;
let str_start =
seg_addr
.checked_add(string_offset as usize)
.checked_add(to_usize(string_offset)?)
.ok_or(FormatError::Overflow(
"local heap seg_addr + string_offset overflow".into(),
))?;
let seg_end = seg_addr
.checked_add(self.data_segment_size as usize)
.checked_add(to_usize(self.data_segment_size)?)
.ok_or(FormatError::Overflow(
"local heap seg_addr + data_segment_size overflow".into(),
))?;
+6 -5
View File
@@ -5,6 +5,7 @@ use alloc::vec::Vec;
use byteorder::{ByteOrder, LittleEndian};
use crate::addr::to_usize;
use crate::error::FormatError;
use crate::message_type::MessageType;
@@ -264,8 +265,8 @@ impl ObjectHeader {
// Follow continuations (v1 continuation chunks are just raw
// messages, no signature); check_message has checked the body.
if msg_type == MessageType::ObjectHeaderContinuation {
let cont_offset = read_offset(body, 0, offset_size)? as usize;
let cont_length = read_offset(body, offset_size as usize, length_size)? as usize;
let cont_offset = to_usize(read_offset(body, 0, offset_size)?)?;
let cont_length = to_usize(read_offset(body, offset_size as usize, length_size)?)?;
Self::parse_v1_chunk(
data,
cont_offset,
@@ -339,7 +340,7 @@ impl ObjectHeader {
_ => unreachable!(),
};
ensure_len(data, pos, chunk_size_width as usize)?;
let chunk0_size = read_offset(data, pos, chunk_size_width)? as usize;
let chunk0_size = to_usize(read_offset(data, pos, chunk_size_width)?)?;
pos += chunk_size_width as usize;
// Bit 2: attribute creation order tracked → messages include creation order field
let has_creation_order = flags & 0x04 != 0;
@@ -472,8 +473,8 @@ impl ObjectHeader {
let msg_type = MessageType::from_u16(msg_type_raw);
if msg_type == MessageType::ObjectHeaderContinuation {
// check_message has checked the body holds both fields.
let cont_off = read_offset(body, 0, offset_size)? as usize;
let cont_len = read_offset(body, offset_size as usize, length_size)? as usize;
let cont_off = to_usize(read_offset(body, 0, offset_size)?)?;
let cont_len = to_usize(read_offset(body, offset_size as usize, length_size)?)?;
continuations.push((cont_off, cont_len));
} else if msg_type == MessageType::Nil {
null_count += 1;
+12 -3
View File
@@ -203,7 +203,12 @@ fn copy_overlap(
};
let (src_strides, out_strides) = (strides(src_shape), strides(box_extent));
let last = rank - 1;
let run = ((hi[last] - lo[last]) as usize) * elem_size;
// Byte offsets into the in-memory buffers; one that does not fit `usize`
// (a 32-bit target) is out of both buffers, like one past their ends.
let bytes = |elements: u64| usize::try_from(elements).ok()?.checked_mul(elem_size);
let Some(run) = bytes(hi[last] - lo[last]) else {
return;
};
let mut idx = lo.clone();
loop {
@@ -213,8 +218,12 @@ fn copy_overlap(
let out_at: u64 = (0..rank)
.map(|d| (idx[d] - box_start[d]) * out_strides[d])
.sum();
let (s, o) = (src_at as usize * elem_size, out_at as usize * elem_size);
if let (Some(from), Some(to)) = (src.get(s..s + run), out.get_mut(o..o + run)) {
if let (Some(s), Some(o)) = (bytes(src_at), bytes(out_at))
&& let (Some(from), Some(to)) = (
src.get(s..s.saturating_add(run)),
out.get_mut(o..o.saturating_add(run)),
)
{
to.copy_from_slice(from);
}
// Advance over every dimension but the last.
+5 -4
View File
@@ -19,6 +19,7 @@ use alloc::{vec, vec::Vec};
use core::ops::Range;
use crate::addr::to_usize;
use crate::error::FormatError;
/// A selection describing which elements of a dataset to access.
@@ -562,7 +563,7 @@ fn decode_hyperslab(r: &mut SelReader, version: u64) -> Result<SerializedSelecti
if !matches!(enc_size, 2 | 4 | 8) {
return Err(sel_err("unsupported hyperslab coordinate encoding size"));
}
let rank = r.uint(4)? as usize;
let rank = to_usize(r.uint(4)?)?;
// HDF5 caps dataspace rank at 32 (H5S_MAX_RANK). Reject anything else so a
// corrupt rank can't drive a huge allocation or read loop.
if rank == 0 || rank > 32 {
@@ -625,11 +626,11 @@ fn decode_hyperslab(r: &mut SelReader, version: u64) -> Result<SerializedSelecti
return Err(FormatError::UnexpectedEof {
expected: r
.pos
.saturating_add(nblocks.saturating_mul(per_block) as usize),
.saturating_add(to_usize(nblocks.saturating_mul(per_block))?),
available: r.data.len(),
});
}
let n = nblocks as usize * rank;
let n = to_usize(nblocks)? * rank;
let (mut starts, mut ends) = (Vec::with_capacity(n), Vec::with_capacity(n));
for _ in 0..nblocks {
for _ in 0..rank {
@@ -662,7 +663,7 @@ fn blocks_union_coords(
.filter(|&t| t <= MAX_EXPANDED_POINTS)
.ok_or_else(|| sel_err("irregular hyperslab selection is too large to expand"))?;
}
let mut out = Vec::with_capacity(total as usize);
let mut out = Vec::with_capacity(to_usize(total)?);
for (s, e) in starts.chunks_exact(rank).zip(ends.chunks_exact(rank)) {
let mut cur = s.to_vec();
'block: loop {
+5 -4
View File
@@ -23,6 +23,7 @@ use alloc::vec::Vec;
#[cfg(feature = "std")]
use std::borrow::Cow;
use crate::addr::to_usize;
use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records};
use crate::error::FormatError;
use crate::fractal_heap::FractalHeapHeader;
@@ -421,7 +422,7 @@ pub fn load_sohm_table(
else {
return Ok(None);
};
let ext = ObjectHeader::parse(file_data, ext_addr as usize, offset_size, length_size)?;
let ext = ObjectHeader::parse(file_data, to_usize(ext_addr)?, offset_size, length_size)?;
let Some(msg) = ext
.messages
.iter()
@@ -432,7 +433,7 @@ pub fn load_sohm_table(
let table_msg = parse_sohm_table_message(&msg.data, offset_size)?;
parse_sohm_table(
file_data,
table_msg.table_address as usize,
to_usize(table_msg.table_address)?,
table_msg.nindexes,
offset_size,
)
@@ -505,7 +506,7 @@ pub fn resolve_sohm_message(
let fh_header = FractalHeapHeader::parse(
file_data,
index.heap_addr as usize,
to_usize(index.heap_addr)?,
offset_size,
length_size,
)?;
@@ -587,7 +588,7 @@ pub fn resolve_shared_message_with_sohm(
) {
(Some(addr), _) => {
let target_header =
ObjectHeader::parse(file_data, addr as usize, offset_size, length_size)?;
ObjectHeader::parse(file_data, to_usize(addr)?, offset_size, length_size)?;
for msg in &target_header.messages {
if msg.msg_type == target_msg_type && !is_shared(msg.flags) {
return Ok(msg.data.clone());
+7 -6
View File
@@ -15,6 +15,7 @@
#[cfg(not(feature = "std"))]
use alloc::{format, string::String, vec, vec::Vec};
use crate::addr::to_usize;
use crate::data_layout::{DataLayout, VdsMapping, parse_vds_mappings};
use crate::dataspace::Dataspace;
use crate::datatype::Datatype;
@@ -208,7 +209,7 @@ fn load_mappings(
return Ok(Vec::new());
};
let coll =
crate::global_heap::GlobalHeapCollection::parse(file_data, addr as usize, length_size)?;
crate::global_heap::GlobalHeapCollection::parse(file_data, to_usize(addr)?, length_size)?;
let index = u16::try_from(*global_heap_index)
.map_err(|_| vds_err("VDS mapping heap index out of range"))?;
let obj = coll
@@ -611,12 +612,12 @@ fn scatter(
return Err(vds_err("virtual/source selection element counts differ"));
}
for (&v, &s) in vidx.iter().zip(sidx) {
let (vo, so) = (v as usize * elem_size, s as usize * elem_size);
let (vo, so) = (to_usize(v)? * elem_size, to_usize(s)? * elem_size);
if vo + elem_size > out.len() || so + elem_size > src.len() {
return Err(vds_err("virtual dataset selection out of bounds"));
}
out[vo..vo + elem_size].copy_from_slice(&src[so..so + elem_size]);
mapped[v as usize] = true;
mapped[to_usize(v)?] = true;
}
Ok(())
}
@@ -747,7 +748,7 @@ fn selection_indices(
return Err(vds_err("VDS selection blocks overlap"));
}
}
let mut out = Vec::with_capacity(volume as usize);
let mut out = Vec::with_capacity(to_usize(volume)?);
for (s, e) in starts.chunks_exact(rank).zip(ends.chunks_exact(rank)) {
let mut cur = s.to_vec();
'block: loop {
@@ -868,7 +869,7 @@ fn load_source_file(whole: &mut [u8]) -> Result<(), FormatError> {
// read as before, up to its length.
let end = sb
.data_end(base as u64, whole.len() as u64)
.map_or(whole.len(), |e| base + e as usize);
.map_or(Ok(whole.len()), |e| to_usize(e).map(|e| base + e))?;
crate::superblock_ext::apply_cache_image_in_place(&mut whole[base..end], &sb)
}
@@ -921,7 +922,7 @@ fn open_source(file_data: &[u8], path: &str) -> Result<Option<OpenSource>, Forma
Err(FormatError::PathNotFound(_)) => return Ok(None),
Err(e) => return Err(e),
};
let header = crate::object_header::ObjectHeader::parse(file_data, addr as usize, os, ls)?;
let header = crate::object_header::ObjectHeader::parse(file_data, to_usize(addr)?, os, ls)?;
let mut src = OpenSource {
offset_size: os,
length_size: ls,
+4 -3
View File
@@ -9,6 +9,7 @@ use alloc::{collections::BTreeMap, format, string::String, vec, vec::Vec};
#[cfg(feature = "std")]
use std::collections::BTreeMap;
use crate::addr::to_usize;
use crate::error::FormatError;
use crate::global_heap::{GlobalHeapCollection, GlobalHeapIndex};
@@ -55,7 +56,7 @@ pub fn parse_vl_references(
) -> Result<Vec<VlElement>, FormatError> {
let elem_size = 4 + offset_size as usize + 4; // length + address + index
let total =
(num_elements as usize)
to_usize(num_elements)?
.checked_mul(elem_size)
.ok_or(FormatError::UnexpectedEof {
expected: usize::MAX,
@@ -68,7 +69,7 @@ pub fn parse_vl_references(
});
}
let mut elements = Vec::with_capacity(num_elements as usize);
let mut elements = Vec::with_capacity(to_usize(num_elements)?);
let mut pos = 0;
for _ in 0..num_elements {
@@ -406,7 +407,7 @@ impl<'a> VlResolver<'a> {
let index =
GlobalHeapCollection::parse_index(self.file_data, offset, self.length_size)?;
// parse_index checked that the collection lies in the file.
let end = offset + index.collection_size as usize;
let end = offset + to_usize(index.collection_size)?;
self.check_overlap(offset, end)?;
let coll = CachedCollection::new(index);
if self.cached_bytes.saturating_add(coll.cost()) > self.budget {
+35
View File
@@ -0,0 +1,35 @@
#!/usr/bin/env bash
# CI check: clawhdf5-format has no truncating `u64 as usize` cast on a 32-bit
# target. HDF5 addresses and lengths are 64-bit; on wasm32 (or any 32-bit
# target) such a cast silently wraps an address past 4 GiB onto another part
# of the file. File values go through `addr::to_usize` (a clean error) and
# in-memory counts through `addr::saturating_usize`.
#
# Lints wasm32-unknown-unknown with clippy's cast_possible_truncation and
# fails on any u64 -> usize finding (other truncations are not checked here).
#
# Usage:
# ./scripts/check-32bit-casts.sh
#
# Prerequisites:
# rustup target add wasm32-unknown-unknown
set -euo pipefail
TARGET="wasm32-unknown-unknown"
echo "==> Checking for truncating u64 -> usize casts in clawhdf5-format ($TARGET)"
out=$(cargo clippy -p clawhdf5-format --target "$TARGET" \
--features plugin-filters --message-format short \
-- -A clippy::all -W clippy::cast_possible_truncation 2>&1) || {
echo "$out"
echo "==> clippy failed" >&2
exit 1
}
found=$(grep -F 'casting `u64` to `usize`' <<<"$out" || true)
if [ -n "$found" ]; then
echo "$found"
echo "==> use addr::to_usize (file values) or addr::saturating_usize (in-memory counts)" >&2
exit 1
fi
echo "==> no truncating u64 -> usize casts"
+2
View File
@@ -147,6 +147,8 @@ run_step "wasm32 clippy (clawhdf5-wasm)" cargo clippy \
--target wasm32-unknown-unknown \
--all-targets \
-- -D warnings
# A 64-bit file address must not wrap on a 32-bit target.
run_step "check-32bit-casts.sh" "$SCRIPT_DIR/check-32bit-casts.sh"
# The built wasm package, run under Node against h5py/netCDF4-written files,
# and the viewer page in headless Chromium when one is found.