harden: make the new readers panic-free on malformed input
The readers added this cycle parse untrusted bytes, so malformed/hostile input must produce errors — never a panic, OOM, or unbounded recursion. Audited each new surface and fixed the concrete vectors, each covered by an adversarial regression test: - Paged Fixed Array: `1 << max_nelmts_bits` shift overflow (u8 up to 255); element-count bounded by file size; element/page offset multiplies checked. - H5S selection decoder: ALL/NONE validate they have the 16 bytes they claim to consume; hyperslab rank capped at 32 (H5S_MAX_RANK); iter_linear coordinate/stride/product arithmetic uses checked ops. - VDS mapping parser: drop pre-allocation from the untrusted `nused`; bounds-check all selection slicing. - scale-offset / N-Bit filters: `1 << minbits` overflow at minbits==64; N-Bit `bit_offset + precision` overflow; N-Bit type-tree recursion depth capped to stop a crafted nested tree from overflowing the stack; element counts bounded by the chunk's expected decompressed size (threaded the previously-unused chunk_size into both decoders) so a bogus count can't over-allocate. - VDS assembly: a virtual dataset whose source is itself virtual (a cycle) now errors instead of recursing into a stack overflow. 16 new adversarial tests; full format suite (482 lib) + agent + facade green; clippy clean. Co-Authored-By: Claude Opus 4.8 <[email protected]>
This commit is contained in:
@@ -2,6 +2,27 @@
|
|||||||
|
|
||||||
## Unreleased
|
## Unreleased
|
||||||
|
|
||||||
|
### Robustness
|
||||||
|
- `clawhdf5-format`: harden the readers added this cycle against malformed /
|
||||||
|
hostile input — they parse untrusted bytes and must return errors, never
|
||||||
|
panic, OOM, or recurse without bound. Fixed concrete vectors found by audit
|
||||||
|
and locked in with adversarial tests:
|
||||||
|
- **Paged Fixed Array**: `1 << max_nelmts_bits` shift overflow (a `u8` ≥ 64);
|
||||||
|
element/page offset multiplications now checked; element count bounded by
|
||||||
|
file size.
|
||||||
|
- **H5S selection decoder**: `ALL`/`NONE` no longer claim 16 bytes they don't
|
||||||
|
have; hyperslab `rank` capped at 32 (`H5S_MAX_RANK`) to stop a giant
|
||||||
|
allocation; `iter_linear` coordinate/stride/product arithmetic is checked.
|
||||||
|
- **VDS mapping parser**: no pre-allocation from the untrusted `nused`; all
|
||||||
|
selection slicing is bounds-checked.
|
||||||
|
- **scale-offset / N-Bit filters**: `1 << minbits` overflow at `minbits == 64`;
|
||||||
|
N-Bit `bit_offset + precision` overflow; N-Bit type-tree recursion depth
|
||||||
|
capped (no stack overflow from a crafted nested tree); element counts
|
||||||
|
bounded by the chunk's expected decompressed size so a bogus count can't
|
||||||
|
drive a huge allocation.
|
||||||
|
- **Virtual Dataset assembly**: a virtual dataset whose source is itself
|
||||||
|
virtual (a cycle) now errors instead of recursing into a stack overflow.
|
||||||
|
|
||||||
### New Features
|
### New Features
|
||||||
- `clawhdf5-agent`: **compress fixed-length string datasets** (memory text
|
- `clawhdf5-agent`: **compress fixed-length string datasets** (memory text
|
||||||
chunks, session summaries, ids, tags, entity/relation names, …). These were
|
chunks, session summaries, ids, tags, entity/relation names, …). These were
|
||||||
|
|||||||
@@ -104,7 +104,29 @@ pub fn parse_vds_mappings(
|
|||||||
let nused = read_length(heap_data, pos, length_size)?;
|
let nused = read_length(heap_data, pos, length_size)?;
|
||||||
pos += ls;
|
pos += ls;
|
||||||
|
|
||||||
let mut mappings = Vec::with_capacity(nused as usize);
|
// `nused` is untrusted; don't pre-allocate from it. Each entry consumes at
|
||||||
|
// least a few bytes, so the loop is naturally bounded by the heap data and
|
||||||
|
// a bogus `nused` simply errors out on the first short read.
|
||||||
|
let mut mappings = Vec::new();
|
||||||
|
// Reads one self-describing selection at `pos`, returning its raw bytes and
|
||||||
|
// advancing past it — bounds-checked so a corrupt selection can't overrun.
|
||||||
|
let read_selection = |heap_data: &[u8], pos: &mut usize| -> Result<Vec<u8>, FormatError> {
|
||||||
|
let rest = heap_data.get(*pos..).ok_or(FormatError::UnexpectedEof {
|
||||||
|
expected: *pos,
|
||||||
|
available: heap_data.len(),
|
||||||
|
})?;
|
||||||
|
let (_, len) = Selection::decode_serialized(rest)?;
|
||||||
|
let bytes = rest
|
||||||
|
.get(..len)
|
||||||
|
.ok_or(FormatError::UnexpectedEof {
|
||||||
|
expected: pos.saturating_add(len),
|
||||||
|
available: heap_data.len(),
|
||||||
|
})?
|
||||||
|
.to_vec();
|
||||||
|
*pos += len;
|
||||||
|
Ok(bytes)
|
||||||
|
};
|
||||||
|
|
||||||
for _ in 0..nused {
|
for _ in 0..nused {
|
||||||
// Source file name (with the version-1 same-file marker handled).
|
// Source file name (with the version-1 same-file marker handled).
|
||||||
let source_file = if version >= 1 && heap_data.get(pos) == Some(&0x04) {
|
let source_file = if version >= 1 && heap_data.get(pos) == Some(&0x04) {
|
||||||
@@ -117,15 +139,9 @@ pub fn parse_vds_mappings(
|
|||||||
// Source dataset name.
|
// Source dataset name.
|
||||||
let source_dataset = read_null_terminated_string(heap_data, &mut pos)?;
|
let source_dataset = read_null_terminated_string(heap_data, &mut pos)?;
|
||||||
|
|
||||||
// Source selection (self-describing length).
|
// Source selection, then virtual selection (both self-describing length).
|
||||||
let (_, ssel_len) = Selection::decode_serialized(&heap_data[pos..])?;
|
let source_selection = read_selection(heap_data, &mut pos)?;
|
||||||
let source_selection = heap_data[pos..pos + ssel_len].to_vec();
|
let virtual_selection = read_selection(heap_data, &mut pos)?;
|
||||||
pos += ssel_len;
|
|
||||||
|
|
||||||
// Virtual selection.
|
|
||||||
let (_, vsel_len) = Selection::decode_serialized(&heap_data[pos..])?;
|
|
||||||
let virtual_selection = heap_data[pos..pos + vsel_len].to_vec();
|
|
||||||
pos += vsel_len;
|
|
||||||
|
|
||||||
mappings.push(VdsMapping {
|
mappings.push(VdsMapping {
|
||||||
source_file,
|
source_file,
|
||||||
@@ -815,4 +831,34 @@ mod tests {
|
|||||||
assert_eq!(mappings[0].source_file, "src_ext.h5");
|
assert_eq!(mappings[0].source_file, "src_ext.h5");
|
||||||
assert_eq!(mappings[0].source_dataset, "data");
|
assert_eq!(mappings[0].source_dataset, "data");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn parse_vds_mappings_huge_nused_does_not_oom_or_panic() {
|
||||||
|
// nused = u64::MAX with no entry data: must error, not pre-allocate or
|
||||||
|
// overrun.
|
||||||
|
let mut blob = vec![0x01u8];
|
||||||
|
blob.extend_from_slice(&u64::MAX.to_le_bytes());
|
||||||
|
assert!(parse_vds_mappings(&blob, 8).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn parse_vds_mappings_truncated_selection_does_not_overrun() {
|
||||||
|
// One entry whose source selection (ALL) is truncated to 8 of 16 bytes.
|
||||||
|
let blob = [
|
||||||
|
0x01u8, // version 1
|
||||||
|
0x01, 0, 0, 0, 0, 0, 0, 0, // nused = 1
|
||||||
|
0x04, // same-file marker
|
||||||
|
0x78, 0x00, // "x\0"
|
||||||
|
0x03, 0, 0, 0, 0x01, 0, 0, 0, // ALL header, truncated (8 of 16 bytes)
|
||||||
|
];
|
||||||
|
assert!(parse_vds_mappings(&blob, 8).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn parse_vds_mappings_empty_is_ok_empty() {
|
||||||
|
assert!(parse_vds_mappings(&[], 8).unwrap().is_empty());
|
||||||
|
// Header present, nused = 0.
|
||||||
|
let blob = [0x01u8, 0, 0, 0, 0, 0, 0, 0, 0];
|
||||||
|
assert!(parse_vds_mappings(&blob, 8).unwrap().is_empty());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -554,6 +554,14 @@ fn read_named_dataset_raw(
|
|||||||
let dl_msg = find(MessageType::DataLayout)
|
let dl_msg = find(MessageType::DataLayout)
|
||||||
.ok_or_else(|| FormatError::ChunkedReadError("VDS source has no data layout".into()))?;
|
.ok_or_else(|| FormatError::ChunkedReadError("VDS source has no data layout".into()))?;
|
||||||
let layout = DataLayout::parse(&dl_msg.data, sb.offset_size, sb.length_size)?;
|
let layout = DataLayout::parse(&dl_msg.data, sb.offset_size, sb.length_size)?;
|
||||||
|
// A virtual dataset whose source is itself another virtual dataset could
|
||||||
|
// form a cycle (A -> B -> A) and recurse into a stack overflow. Nested
|
||||||
|
// virtual sources are exotic and unsupported, so stop here cleanly.
|
||||||
|
if matches!(layout, DataLayout::Virtual { .. }) {
|
||||||
|
return Err(FormatError::ChunkedReadError(
|
||||||
|
"virtual dataset source is itself virtual (unsupported)".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
let pipeline = find(MessageType::FilterPipeline)
|
let pipeline = find(MessageType::FilterPipeline)
|
||||||
.map(|m| FilterPipeline::parse(&m.data))
|
.map(|m| FilterPipeline::parse(&m.data))
|
||||||
.transpose()?;
|
.transpose()?;
|
||||||
|
|||||||
@@ -17,7 +17,7 @@ use crate::filter_pipeline::{
|
|||||||
pub fn decompress_chunk(
|
pub fn decompress_chunk(
|
||||||
compressed: &[u8],
|
compressed: &[u8],
|
||||||
pipeline: &FilterPipeline,
|
pipeline: &FilterPipeline,
|
||||||
_chunk_size: usize,
|
chunk_size: usize,
|
||||||
element_size: u32,
|
element_size: u32,
|
||||||
) -> Result<Vec<u8>, FormatError> {
|
) -> Result<Vec<u8>, FormatError> {
|
||||||
let mut data = compressed.to_vec();
|
let mut data = compressed.to_vec();
|
||||||
@@ -29,8 +29,10 @@ pub fn decompress_chunk(
|
|||||||
FILTER_LZ4 => lz4_decompress(&data)?,
|
FILTER_LZ4 => lz4_decompress(&data)?,
|
||||||
FILTER_ZSTD => zstd_decompress(&data)?,
|
FILTER_ZSTD => zstd_decompress(&data)?,
|
||||||
FILTER_FLETCHER32 => fletcher32_verify(&data)?,
|
FILTER_FLETCHER32 => fletcher32_verify(&data)?,
|
||||||
FILTER_SCALEOFFSET => scaleoffset_decompress(&data, &filter.client_data)?,
|
// `chunk_size` is the expected decompressed size; pass it so these
|
||||||
FILTER_NBIT => nbit_decompress(&data, &filter.client_data)?,
|
// decoders can reject an element count that would over-allocate.
|
||||||
|
FILTER_SCALEOFFSET => scaleoffset_decompress(&data, &filter.client_data, chunk_size)?,
|
||||||
|
FILTER_NBIT => nbit_decompress(&data, &filter.client_data, chunk_size)?,
|
||||||
other => return Err(FormatError::UnsupportedFilter(other)),
|
other => return Err(FormatError::UnsupportedFilter(other)),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -86,7 +88,11 @@ pub fn compress_chunk(
|
|||||||
/// (0 = float D-scale, 2 = integer), `[1]`=scale factor (decimal digits for
|
/// (0 = float D-scale, 2 = integer), `[1]`=scale factor (decimal digits for
|
||||||
/// D-scale), `[2]`=element count, `[4]`=element size, `[5]`=signed flag,
|
/// D-scale), `[2]`=element count, `[4]`=element size, `[5]`=signed flag,
|
||||||
/// `[6]`=byte order (1 = big-endian), `[7]`=fill defined, `[8..]`=fill value.
|
/// `[6]`=byte order (1 = big-endian), `[7]`=fill defined, `[8..]`=fill value.
|
||||||
fn scaleoffset_decompress(data: &[u8], cd: &[u32]) -> Result<Vec<u8>, FormatError> {
|
fn scaleoffset_decompress(
|
||||||
|
data: &[u8],
|
||||||
|
cd: &[u32],
|
||||||
|
expected_bytes: usize,
|
||||||
|
) -> Result<Vec<u8>, FormatError> {
|
||||||
const H5Z_SO_FLOAT_DSCALE: u32 = 0;
|
const H5Z_SO_FLOAT_DSCALE: u32 = 0;
|
||||||
const H5Z_SO_INT: u32 = 2;
|
const H5Z_SO_INT: u32 = 2;
|
||||||
if cd.len() < 8 {
|
if cd.len() < 8 {
|
||||||
@@ -107,6 +113,17 @@ fn scaleoffset_decompress(data: &[u8], cd: &[u32]) -> Result<Vec<u8>, FormatErro
|
|||||||
"scale-offset: unsupported element size".into(),
|
"scale-offset: unsupported element size".into(),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
// The decoded output must match the chunk's uncompressed size; reject an
|
||||||
|
// element count that would over-allocate (e.g. minbits == 0 with a huge
|
||||||
|
// nelmts and no packed payload to bound it).
|
||||||
|
let out_bytes = nelmts
|
||||||
|
.checked_mul(elem_size)
|
||||||
|
.ok_or_else(|| FormatError::ChunkedReadError("scale-offset: size overflow".into()))?;
|
||||||
|
if expected_bytes != 0 && out_bytes > expected_bytes {
|
||||||
|
return Err(FormatError::ChunkedReadError(
|
||||||
|
"scale-offset: element count exceeds chunk size".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
let signed = cd[5] == 1;
|
let signed = cd[5] == 1;
|
||||||
let big_endian = cd[6] == 1;
|
let big_endian = cd[6] == 1;
|
||||||
let fill_defined = cd[7] == 1;
|
let fill_defined = cd[7] == 1;
|
||||||
@@ -163,7 +180,14 @@ fn scaleoffset_decompress(data: &[u8], cd: &[u32]) -> Result<Vec<u8>, FormatErro
|
|||||||
};
|
};
|
||||||
// The fill code (all ones) only exists when there are bits to pack.
|
// The fill code (all ones) only exists when there are bits to pack.
|
||||||
let has_fill_code = fill_defined && minbits > 0 && minbits < 64;
|
let has_fill_code = fill_defined && minbits > 0 && minbits < 64;
|
||||||
let fill_code: u64 = if minbits == 0 { 0 } else { (1u64 << minbits) - 1 };
|
// Computed for all 1..=64 widths; `1 << 64` would overflow, so saturate.
|
||||||
|
let fill_code: u64 = if minbits == 0 {
|
||||||
|
0
|
||||||
|
} else if minbits >= 64 {
|
||||||
|
u64::MAX
|
||||||
|
} else {
|
||||||
|
(1u64 << minbits) - 1
|
||||||
|
};
|
||||||
|
|
||||||
if is_float {
|
if is_float {
|
||||||
let scale = 10f64.powi(cd[1] as i32);
|
let scale = 10f64.powi(cd[1] as i32);
|
||||||
@@ -339,11 +363,20 @@ fn nbit_cd(cd: &[u32], i: usize) -> Result<u32, FormatError> {
|
|||||||
.ok_or_else(|| FormatError::ChunkedReadError("nbit: truncated client data".into()))
|
.ok_or_else(|| FormatError::ChunkedReadError("nbit: truncated client data".into()))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Maximum N-Bit type-tree nesting depth. Real types nest only a few levels;
|
||||||
|
/// the bound stops a crafted tree from recursing into a stack overflow.
|
||||||
|
const NBIT_MAX_DEPTH: u32 = 64;
|
||||||
|
|
||||||
/// Parse one N-Bit type node from the client-data tree, advancing `idx`.
|
/// Parse one N-Bit type node from the client-data tree, advancing `idx`.
|
||||||
fn parse_nbit_node(cd: &[u32], idx: &mut usize) -> Result<NbitNode, FormatError> {
|
fn parse_nbit_node(cd: &[u32], idx: &mut usize, depth: u32) -> Result<NbitNode, FormatError> {
|
||||||
const ATOMIC: u32 = 1;
|
const ATOMIC: u32 = 1;
|
||||||
const ARRAY: u32 = 2;
|
const ARRAY: u32 = 2;
|
||||||
const COMPOUND: u32 = 3;
|
const COMPOUND: u32 = 3;
|
||||||
|
if depth > NBIT_MAX_DEPTH {
|
||||||
|
return Err(FormatError::ChunkedReadError(
|
||||||
|
"nbit: type tree nested too deeply".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
let class = nbit_cd(cd, *idx)?;
|
let class = nbit_cd(cd, *idx)?;
|
||||||
match class {
|
match class {
|
||||||
ATOMIC => {
|
ATOMIC => {
|
||||||
@@ -353,7 +386,11 @@ fn parse_nbit_node(cd: &[u32], idx: &mut usize) -> Result<NbitNode, FormatError>
|
|||||||
let precision = nbit_cd(cd, *idx + 3)?;
|
let precision = nbit_cd(cd, *idx + 3)?;
|
||||||
let bit_offset = nbit_cd(cd, *idx + 4)?;
|
let bit_offset = nbit_cd(cd, *idx + 4)?;
|
||||||
*idx += 5;
|
*idx += 5;
|
||||||
if size == 0 || size > 8 || precision == 0 || bit_offset + precision > (size * 8) as u32
|
let end_bit = bit_offset.checked_add(precision);
|
||||||
|
if size == 0
|
||||||
|
|| size > 8
|
||||||
|
|| precision == 0
|
||||||
|
|| end_bit.is_none_or(|e| e > (size * 8) as u32)
|
||||||
{
|
{
|
||||||
return Err(FormatError::ChunkedReadError(
|
return Err(FormatError::ChunkedReadError(
|
||||||
"nbit: invalid atomic parameters".into(),
|
"nbit: invalid atomic parameters".into(),
|
||||||
@@ -370,7 +407,7 @@ fn parse_nbit_node(cd: &[u32], idx: &mut usize) -> Result<NbitNode, FormatError>
|
|||||||
// class, total size, base type node
|
// class, total size, base type node
|
||||||
let total = nbit_cd(cd, *idx + 1)? as usize;
|
let total = nbit_cd(cd, *idx + 1)? as usize;
|
||||||
*idx += 2;
|
*idx += 2;
|
||||||
let base = parse_nbit_node(cd, idx)?;
|
let base = parse_nbit_node(cd, idx, depth + 1)?;
|
||||||
let base_size = base.byte_size();
|
let base_size = base.byte_size();
|
||||||
if base_size == 0 || !total.is_multiple_of(base_size) {
|
if base_size == 0 || !total.is_multiple_of(base_size) {
|
||||||
return Err(FormatError::ChunkedReadError(
|
return Err(FormatError::ChunkedReadError(
|
||||||
@@ -388,12 +425,17 @@ fn parse_nbit_node(cd: &[u32], idx: &mut usize) -> Result<NbitNode, FormatError>
|
|||||||
let total = nbit_cd(cd, *idx + 1)? as usize;
|
let total = nbit_cd(cd, *idx + 1)? as usize;
|
||||||
let nmembers = nbit_cd(cd, *idx + 2)? as usize;
|
let nmembers = nbit_cd(cd, *idx + 2)? as usize;
|
||||||
*idx += 3;
|
*idx += 3;
|
||||||
let mut members = Vec::with_capacity(nmembers);
|
// Don't pre-allocate from the untrusted member count; the loop is
|
||||||
|
// bounded by `nbit_cd` running out of client data.
|
||||||
|
let mut members = Vec::new();
|
||||||
for _ in 0..nmembers {
|
for _ in 0..nmembers {
|
||||||
let moff = nbit_cd(cd, *idx)? as usize;
|
let moff = nbit_cd(cd, *idx)? as usize;
|
||||||
*idx += 1;
|
*idx += 1;
|
||||||
let node = parse_nbit_node(cd, idx)?;
|
let node = parse_nbit_node(cd, idx, depth + 1)?;
|
||||||
if moff + node.byte_size() > total {
|
if moff
|
||||||
|
.checked_add(node.byte_size())
|
||||||
|
.is_none_or(|end| end > total)
|
||||||
|
{
|
||||||
return Err(FormatError::ChunkedReadError(
|
return Err(FormatError::ChunkedReadError(
|
||||||
"nbit: member exceeds compound size".into(),
|
"nbit: member exceeds compound size".into(),
|
||||||
));
|
));
|
||||||
@@ -488,7 +530,7 @@ fn decode_nbit_node(
|
|||||||
/// (HDF5's canonical reduced-precision layout). Sign-extension of reduced
|
/// (HDF5's canonical reduced-precision layout). Sign-extension of reduced
|
||||||
/// precision signed integers is the datatype reader's job. Atomic floats are
|
/// precision signed integers is the datatype reader's job. Atomic floats are
|
||||||
/// encoded as full-precision atomics and handled transparently.
|
/// encoded as full-precision atomics and handled transparently.
|
||||||
fn nbit_decompress(data: &[u8], cd: &[u32]) -> Result<Vec<u8>, FormatError> {
|
fn nbit_decompress(data: &[u8], cd: &[u32], expected_bytes: usize) -> Result<Vec<u8>, FormatError> {
|
||||||
if cd.len() < 4 {
|
if cd.len() < 4 {
|
||||||
return Err(FormatError::ChunkedReadError(
|
return Err(FormatError::ChunkedReadError(
|
||||||
"nbit: missing filter client data".into(),
|
"nbit: missing filter client data".into(),
|
||||||
@@ -496,7 +538,7 @@ fn nbit_decompress(data: &[u8], cd: &[u32]) -> Result<Vec<u8>, FormatError> {
|
|||||||
}
|
}
|
||||||
let nelmts = cd[2] as usize;
|
let nelmts = cd[2] as usize;
|
||||||
let mut idx = 3;
|
let mut idx = 3;
|
||||||
let root = parse_nbit_node(cd, &mut idx)?;
|
let root = parse_nbit_node(cd, &mut idx, 0)?;
|
||||||
let elem_size = root.byte_size();
|
let elem_size = root.byte_size();
|
||||||
if elem_size == 0 {
|
if elem_size == 0 {
|
||||||
return Err(FormatError::ChunkedReadError(
|
return Err(FormatError::ChunkedReadError(
|
||||||
@@ -507,6 +549,13 @@ fn nbit_decompress(data: &[u8], cd: &[u32]) -> Result<Vec<u8>, FormatError> {
|
|||||||
let total = nelmts
|
let total = nelmts
|
||||||
.checked_mul(elem_size)
|
.checked_mul(elem_size)
|
||||||
.ok_or_else(|| FormatError::ChunkedReadError("nbit: size overflow".into()))?;
|
.ok_or_else(|| FormatError::ChunkedReadError("nbit: size overflow".into()))?;
|
||||||
|
// The decoded output must match the chunk's uncompressed size; reject a
|
||||||
|
// count that would over-allocate.
|
||||||
|
if expected_bytes != 0 && total > expected_bytes {
|
||||||
|
return Err(FormatError::ChunkedReadError(
|
||||||
|
"nbit: element count exceeds chunk size".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
let mut out = vec![0u8; total];
|
let mut out = vec![0u8; total];
|
||||||
let mut br = BitReader { data, pos: 0 };
|
let mut br = BitReader { data, pos: 0 };
|
||||||
for elem in out.chunks_exact_mut(elem_size) {
|
for elem in out.chunks_exact_mut(elem_size) {
|
||||||
@@ -1176,7 +1225,7 @@ mod tests {
|
|||||||
0x02, 0x00, 0x00, 0x00, 0x08, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
|
0x02, 0x00, 0x00, 0x00, 0x08, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
|
||||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc6, 0x00,
|
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc6, 0x00,
|
||||||
];
|
];
|
||||||
assert_eq!(scaleoffset_decompress(&raw, &cd).unwrap(), i32_le(&[0, 1, 2, 3]));
|
assert_eq!(scaleoffset_decompress(&raw, &cd, 0).unwrap(), i32_le(&[0, 1, 2, 3]));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -1188,7 +1237,7 @@ mod tests {
|
|||||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x4f, 0x79, 0xce, 0x00,
|
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x4f, 0x79, 0xce, 0x00,
|
||||||
];
|
];
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
scaleoffset_decompress(&raw, &cd).unwrap(),
|
scaleoffset_decompress(&raw, &cd, 0).unwrap(),
|
||||||
i32_le(&[-5, -3, -1, 0, 2, 4, 7, 9])
|
i32_le(&[-5, -3, -1, 0, 2, 4, 7, 9])
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -1202,7 +1251,7 @@ mod tests {
|
|||||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x23, 0x45, 0x67, 0x89, 0x00,
|
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x23, 0x45, 0x67, 0x89, 0x00,
|
||||||
];
|
];
|
||||||
let expected: Vec<u8> = (100u32..110).flat_map(|v| v.to_le_bytes()).collect();
|
let expected: Vec<u8> = (100u32..110).flat_map(|v| v.to_le_bytes()).collect();
|
||||||
assert_eq!(scaleoffset_decompress(&raw, &cd).unwrap(), expected);
|
assert_eq!(scaleoffset_decompress(&raw, &cd, 0).unwrap(), expected);
|
||||||
}
|
}
|
||||||
|
|
||||||
fn as_f32(bytes: &[u8]) -> Vec<f32> {
|
fn as_f32(bytes: &[u8]) -> Vec<f32> {
|
||||||
@@ -1220,7 +1269,7 @@ mod tests {
|
|||||||
0x05, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x80, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x00,
|
0x05, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x80, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x00,
|
||||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x15, 0x40,
|
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x15, 0x40,
|
||||||
];
|
];
|
||||||
let got = as_f32(&scaleoffset_decompress(&raw, &cd).unwrap());
|
let got = as_f32(&scaleoffset_decompress(&raw, &cd, 0).unwrap());
|
||||||
assert_eq!(got, vec![0.0, 1.0, 2.0, 3.0]);
|
assert_eq!(got, vec![0.0, 1.0, 2.0, 3.0]);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1232,7 +1281,7 @@ mod tests {
|
|||||||
0x08, 0x00, 0x00, 0x00, 0x08, 0xcd, 0xcc, 0xcc, 0x3d, 0x00, 0x00, 0x00, 0x00, 0x00,
|
0x08, 0x00, 0x00, 0x00, 0x08, 0xcd, 0xcc, 0xcc, 0x3d, 0x00, 0x00, 0x00, 0x00, 0x00,
|
||||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0x00, 0x64, 0xc8, 0x00,
|
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0x00, 0x64, 0xc8, 0x00,
|
||||||
];
|
];
|
||||||
let got = as_f32(&scaleoffset_decompress(&raw, &cd).unwrap());
|
let got = as_f32(&scaleoffset_decompress(&raw, &cd, 0).unwrap());
|
||||||
let exp = [0.0f32, 0.1, 0.2, 0.3];
|
let exp = [0.0f32, 0.1, 0.2, 0.3];
|
||||||
assert_eq!(got.len(), 4);
|
assert_eq!(got.len(), 4);
|
||||||
for (g, e) in got.iter().zip(exp.iter()) {
|
for (g, e) in got.iter().zip(exp.iter()) {
|
||||||
@@ -1246,7 +1295,7 @@ mod tests {
|
|||||||
let cd = [1u32, 3, 50, 1, 4, 0, 0, 1, 0];
|
let cd = [1u32, 3, 50, 1, 4, 0, 0, 1, 0];
|
||||||
let raw = [0u8; 24];
|
let raw = [0u8; 24];
|
||||||
assert!(matches!(
|
assert!(matches!(
|
||||||
scaleoffset_decompress(&raw, &cd),
|
scaleoffset_decompress(&raw, &cd, 0),
|
||||||
Err(FormatError::UnsupportedFilter(FILTER_SCALEOFFSET))
|
Err(FormatError::UnsupportedFilter(FILTER_SCALEOFFSET))
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
@@ -1265,7 +1314,7 @@ mod tests {
|
|||||||
.iter()
|
.iter()
|
||||||
.flat_map(|v| v.to_le_bytes())
|
.flat_map(|v| v.to_le_bytes())
|
||||||
.collect();
|
.collect();
|
||||||
assert_eq!(nbit_decompress(&raw, &cd).unwrap(), expected);
|
assert_eq!(nbit_decompress(&raw, &cd, 0).unwrap(), expected);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -1282,7 +1331,7 @@ mod tests {
|
|||||||
0x64, 0x00, 0x00, 0x00, // 0x00000064
|
0x64, 0x00, 0x00, 0x00, // 0x00000064
|
||||||
0x18, 0xfc, 0x00, 0x00, // 0x0000fc18
|
0x18, 0xfc, 0x00, 0x00, // 0x0000fc18
|
||||||
];
|
];
|
||||||
assert_eq!(nbit_decompress(&raw, &cd).unwrap(), expected);
|
assert_eq!(nbit_decompress(&raw, &cd, 0).unwrap(), expected);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -1297,7 +1346,7 @@ mod tests {
|
|||||||
0xe8,0x03,0x00,0x00, 0x07,0x00,0x00,0x00, // (1000, 7)
|
0xe8,0x03,0x00,0x00, 0x07,0x00,0x00,0x00, // (1000, 7)
|
||||||
0x00,0x80,0x00,0x00, 0xff,0x00,0x00,0x00, // (-32768, 255)
|
0x00,0x80,0x00,0x00, 0xff,0x00,0x00,0x00, // (-32768, 255)
|
||||||
];
|
];
|
||||||
assert_eq!(nbit_decompress(&raw, &cd).unwrap(), expected);
|
assert_eq!(nbit_decompress(&raw, &cd, 0).unwrap(), expected);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -1311,7 +1360,7 @@ mod tests {
|
|||||||
0xff,0xff,0x00,0x00, 0x64,0x00,0x00,0x00, 0xc8,0x00,0x00,0x00, // ([-1,100], 200)
|
0xff,0xff,0x00,0x00, 0x64,0x00,0x00,0x00, 0xc8,0x00,0x00,0x00, // ([-1,100], 200)
|
||||||
0xe8,0x03,0x00,0x00, 0x00,0x80,0x00,0x00, 0x07,0x00,0x00,0x00, // ([1000,-32768], 7)
|
0xe8,0x03,0x00,0x00, 0x00,0x80,0x00,0x00, 0x07,0x00,0x00,0x00, // ([1000,-32768], 7)
|
||||||
];
|
];
|
||||||
assert_eq!(nbit_decompress(&raw, &cd).unwrap(), expected);
|
assert_eq!(nbit_decompress(&raw, &cd, 0).unwrap(), expected);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -1328,6 +1377,72 @@ mod tests {
|
|||||||
0xff,0xff,0x00,0x00, 0x00,0x00,0xc0,0x3f, // (-1, 1.5)
|
0xff,0xff,0x00,0x00, 0x00,0x00,0xc0,0x3f, // (-1, 1.5)
|
||||||
0x64,0x00,0x00,0x00, 0x00,0x00,0x20,0xc0, // (100, -2.5)
|
0x64,0x00,0x00,0x00, 0x00,0x00,0x20,0xc0, // (100, -2.5)
|
||||||
];
|
];
|
||||||
assert_eq!(nbit_decompress(&raw, &cd).unwrap(), expected);
|
assert_eq!(nbit_decompress(&raw, &cd, 0).unwrap(), expected);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ----- Adversarial / hardening: malformed filter data must not panic -----
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn scaleoffset_minbits_64_does_not_panic() {
|
||||||
|
// minbits == 64 previously overflowed `1u64 << minbits` computing the
|
||||||
|
// fill code. cd: int, nelmts=1, elem_size=8, fill_defined=1.
|
||||||
|
let cd = [2u32, 0, 1, 0, 8, 1, 0, 1];
|
||||||
|
let mut data = Vec::new();
|
||||||
|
data.extend_from_slice(&64u32.to_le_bytes()); // minbits = 64
|
||||||
|
data.push(8); // minval_width
|
||||||
|
data.extend_from_slice(&0i64.to_le_bytes()); // minval
|
||||||
|
data.extend_from_slice(&[0u8; 8]); // reserved
|
||||||
|
data.extend_from_slice(&[0u8; 8]); // one 64-bit code
|
||||||
|
// Must return a Result (Ok or Err) without panicking.
|
||||||
|
let _ = scaleoffset_decompress(&data, &cd, 8);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn scaleoffset_huge_nelmts_bounded_by_chunk_size() {
|
||||||
|
// minbits == 0 (no packed payload) with a giant nelmts must not try to
|
||||||
|
// allocate when the expected chunk size is small.
|
||||||
|
let cd = [2u32, 0, u32::MAX, 0, 4, 0, 0, 0];
|
||||||
|
let mut data = Vec::new();
|
||||||
|
data.extend_from_slice(&0u32.to_le_bytes()); // minbits = 0
|
||||||
|
data.push(4);
|
||||||
|
data.extend_from_slice(&[0u8; 4]);
|
||||||
|
assert!(scaleoffset_decompress(&data, &cd, 64).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn nbit_deeply_nested_array_is_rejected() {
|
||||||
|
// A chain of ARRAY nodes (class 2) far deeper than NBIT_MAX_DEPTH must
|
||||||
|
// error rather than recurse into a stack overflow. Layout per level:
|
||||||
|
// [class=2, total]. Terminate with a (never-reached) atomic.
|
||||||
|
let mut cd = vec![0u32, 0, 1]; // nparms, flag, nelmts
|
||||||
|
for _ in 0..500 {
|
||||||
|
cd.push(2); // ARRAY
|
||||||
|
cd.push(8); // total size
|
||||||
|
}
|
||||||
|
cd.extend_from_slice(&[1, 8, 0, 8, 0]); // atomic leaf
|
||||||
|
assert!(nbit_decompress(&[], &cd, 0).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn nbit_atomic_bit_offset_overflow_is_rejected() {
|
||||||
|
// bit_offset + precision near u32::MAX must not overflow the check.
|
||||||
|
let cd = [0u32, 0, 1, 1, 8, 0, u32::MAX, u32::MAX];
|
||||||
|
assert!(nbit_decompress(&[0u8; 8], &cd, 0).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn nbit_huge_nelmts_bounded_by_chunk_size() {
|
||||||
|
// Valid 1-byte atomic but an enormous element count; the expected chunk
|
||||||
|
// size bounds the allocation.
|
||||||
|
let cd = [0u32, 0, u32::MAX, 1, 1, 0, 8, 0];
|
||||||
|
assert!(nbit_decompress(&[0u8; 4], &cd, 16).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn scaleoffset_truncated_inputs_do_not_panic() {
|
||||||
|
assert!(scaleoffset_decompress(&[], &[2, 0, 1, 0, 4, 0, 0, 0], 4).is_err());
|
||||||
|
assert!(scaleoffset_decompress(&[0u8; 3], &[2, 0, 1, 0, 4, 0, 0, 0], 4).is_err());
|
||||||
|
// Missing client data entirely.
|
||||||
|
assert!(scaleoffset_decompress(&[0u8; 32], &[2, 0], 4).is_err());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -144,12 +144,30 @@ pub fn read_fixed_array_chunks(
|
|||||||
let elements_start = db_offset + db_header_size;
|
let elements_start = db_offset + db_header_size;
|
||||||
|
|
||||||
let num_elements = header.num_elements as usize;
|
let num_elements = header.num_elements as usize;
|
||||||
|
// A chunk index cannot describe more elements than the file has bytes (each
|
||||||
|
// element occupies at least `offset_size` bytes). Reject a corrupt count
|
||||||
|
// before it can drive a huge loop or overflow an offset computation.
|
||||||
|
if num_elements > file_data.len() {
|
||||||
|
return Err(FormatError::ChunkedReadError(
|
||||||
|
"Fixed Array element count exceeds file size".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
let os = offset_size as usize;
|
let os = offset_size as usize;
|
||||||
// On-disk stride of one element. For non-filtered arrays the element is just
|
// On-disk stride of one element. For non-filtered arrays the element is just
|
||||||
// the chunk address (== offset_size); for filtered arrays it is
|
// the chunk address (== offset_size); for filtered arrays it is
|
||||||
// address + chunk_size + filter_mask (== header.element_size).
|
// address + chunk_size + filter_mask (== header.element_size).
|
||||||
let elem_stride = (header.element_size as usize).max(os);
|
let elem_stride = (header.element_size as usize).max(os);
|
||||||
|
|
||||||
|
// Absolute file offset of element `idx` within a run starting at `base`,
|
||||||
|
// with overflow surfaced as a clean error rather than a panic/wrap.
|
||||||
|
let elem_at = |base: usize, idx: usize| -> Result<usize, FormatError> {
|
||||||
|
idx.checked_mul(elem_stride)
|
||||||
|
.and_then(|o| base.checked_add(o))
|
||||||
|
.ok_or(FormatError::ChunkedReadError(
|
||||||
|
"Fixed Array element offset overflow".into(),
|
||||||
|
))
|
||||||
|
};
|
||||||
|
|
||||||
// Compute chunk offsets based on index.
|
// Compute chunk offsets based on index.
|
||||||
// Chunks are stored in row-major order within the dataset space.
|
// Chunks are stored in row-major order within the dataset space.
|
||||||
let mut num_chunks_per_dim = Vec::with_capacity(rank);
|
let mut num_chunks_per_dim = Vec::with_capacity(rank);
|
||||||
@@ -189,6 +207,13 @@ pub fn read_fixed_array_chunks(
|
|||||||
};
|
};
|
||||||
|
|
||||||
// A data block is paged when it holds more elements than fit in one page.
|
// A data block is paged when it holds more elements than fit in one page.
|
||||||
|
// `max_nelmts_bits` is an untrusted u8; a shift >= the pointer width would
|
||||||
|
// panic, so reject it (real page-size bits are tiny — 10 by default).
|
||||||
|
if header.max_nelmts_bits as u32 >= usize::BITS {
|
||||||
|
return Err(FormatError::ChunkedReadError(
|
||||||
|
"Fixed Array max_nelmts_bits too large".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
let page_nelmts = 1usize << header.max_nelmts_bits;
|
let page_nelmts = 1usize << header.max_nelmts_bits;
|
||||||
let is_paged = num_elements > page_nelmts;
|
let is_paged = num_elements > page_nelmts;
|
||||||
|
|
||||||
@@ -196,7 +221,7 @@ pub fn read_fixed_array_chunks(
|
|||||||
// Non-paged: prefix, then `num_elements` elements packed directly,
|
// Non-paged: prefix, then `num_elements` elements packed directly,
|
||||||
// then a trailing checksum (which we don't validate).
|
// then a trailing checksum (which we don't validate).
|
||||||
for i in 0..num_elements {
|
for i in 0..num_elements {
|
||||||
push_element(i, elements_start + i * elem_stride, &mut chunks)?;
|
push_element(i, elem_at(elements_start, i)?, &mut chunks)?;
|
||||||
}
|
}
|
||||||
return Ok(chunks);
|
return Ok(chunks);
|
||||||
}
|
}
|
||||||
@@ -207,12 +232,18 @@ pub fn read_fixed_array_chunks(
|
|||||||
// only the final page holds fewer elements. Uninitialized pages (bit clear)
|
// only the final page holds fewer elements. Uninitialized pages (bit clear)
|
||||||
// still occupy their slot on disk but are zero-filled, so the bitmap — not a
|
// still occupy their slot on disk but are zero-filled, so the bitmap — not a
|
||||||
// 0xFF sentinel — is what marks a whole page as unallocated.
|
// 0xFF sentinel — is what marks a whole page as unallocated.
|
||||||
|
let stride_overflow = || {
|
||||||
|
FormatError::ChunkedReadError("Fixed Array page offset overflow".into())
|
||||||
|
};
|
||||||
let npages = num_elements.div_ceil(page_nelmts);
|
let npages = num_elements.div_ceil(page_nelmts);
|
||||||
let bitmap_size = npages.div_ceil(8);
|
let bitmap_size = npages.div_ceil(8);
|
||||||
let bitmap_start = elements_start;
|
let bitmap_start = elements_start;
|
||||||
// prefix(db_header_size) + bitmap + checksum(4)
|
// prefix(db_header_size) + bitmap + checksum(4)
|
||||||
let pages_start = db_offset + db_header_size + bitmap_size + 4;
|
let pages_start = db_offset + db_header_size + bitmap_size + 4;
|
||||||
let page_stride = page_nelmts * elem_stride + 4;
|
let page_stride = page_nelmts
|
||||||
|
.checked_mul(elem_stride)
|
||||||
|
.and_then(|x| x.checked_add(4))
|
||||||
|
.ok_or_else(stride_overflow)?;
|
||||||
|
|
||||||
if bitmap_start + bitmap_size > file_data.len() {
|
if bitmap_start + bitmap_size > file_data.len() {
|
||||||
return Err(FormatError::UnexpectedEof {
|
return Err(FormatError::UnexpectedEof {
|
||||||
@@ -222,7 +253,7 @@ pub fn read_fixed_array_chunks(
|
|||||||
}
|
}
|
||||||
|
|
||||||
for p in 0..npages {
|
for p in 0..npages {
|
||||||
let page_first = p * page_nelmts;
|
let page_first = p * page_nelmts; // < num_elements, cannot overflow
|
||||||
let page_count = core::cmp::min(page_nelmts, num_elements - page_first);
|
let page_count = core::cmp::min(page_nelmts, num_elements - page_first);
|
||||||
|
|
||||||
// Check the page-init bit (MSB-first within each byte).
|
// Check the page-init bit (MSB-first within each byte).
|
||||||
@@ -232,9 +263,12 @@ pub fn read_fixed_array_chunks(
|
|||||||
continue; // entire page unallocated
|
continue; // entire page unallocated
|
||||||
}
|
}
|
||||||
|
|
||||||
let page_off = pages_start + p * page_stride;
|
let page_off = p
|
||||||
|
.checked_mul(page_stride)
|
||||||
|
.and_then(|o| pages_start.checked_add(o))
|
||||||
|
.ok_or_else(stride_overflow)?;
|
||||||
for e in 0..page_count {
|
for e in 0..page_count {
|
||||||
push_element(page_first + e, page_off + e * elem_stride, &mut chunks)?;
|
push_element(page_first + e, elem_at(page_off, e)?, &mut chunks)?;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -420,6 +454,41 @@ mod tests {
|
|||||||
assert!(result.is_err());
|
assert!(result.is_err());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Malformed headers must error, never panic (shift overflow, huge counts).
|
||||||
|
#[test]
|
||||||
|
fn read_rejects_oversized_max_nelmts_bits() {
|
||||||
|
let mut buf = vec![0u8; 512];
|
||||||
|
let fahd = 0x40usize;
|
||||||
|
buf[fahd..fahd + 4].copy_from_slice(b"FAHD");
|
||||||
|
buf[fahd + 4] = 0; // version
|
||||||
|
buf[fahd + 5] = 0; // client_id
|
||||||
|
buf[fahd + 6] = 8; // element_size
|
||||||
|
buf[fahd + 7] = 200; // max_nelmts_bits — absurd, would overflow a shift
|
||||||
|
buf[fahd + 8..fahd + 16].copy_from_slice(&3u64.to_le_bytes()); // num_elements
|
||||||
|
buf[fahd + 16..fahd + 24].copy_from_slice(&0x100u64.to_le_bytes());
|
||||||
|
// FADB so parsing reaches the paged check
|
||||||
|
let db = 0x100usize;
|
||||||
|
buf[db..db + 4].copy_from_slice(b"FADB");
|
||||||
|
let header = FixedArrayHeader::parse(&buf, fahd, 8, 8).unwrap();
|
||||||
|
let r = read_fixed_array_chunks(&buf, &header, &[100], &[20], 8, 8, 8);
|
||||||
|
assert!(r.is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn read_rejects_num_elements_larger_than_file() {
|
||||||
|
let mut buf = vec![0u8; 256];
|
||||||
|
let fahd = 0x40usize;
|
||||||
|
buf[fahd..fahd + 4].copy_from_slice(b"FAHD");
|
||||||
|
buf[fahd + 6] = 8;
|
||||||
|
buf[fahd + 7] = 10;
|
||||||
|
buf[fahd + 8..fahd + 16].copy_from_slice(&u64::MAX.to_le_bytes()); // absurd count
|
||||||
|
buf[fahd + 16..fahd + 24].copy_from_slice(&0x80u64.to_le_bytes());
|
||||||
|
buf[0x80..0x84].copy_from_slice(b"FADB");
|
||||||
|
let header = FixedArrayHeader::parse(&buf, fahd, 8, 8).unwrap();
|
||||||
|
let r = read_fixed_array_chunks(&buf, &header, &[100], &[20], 8, 8, 8);
|
||||||
|
assert!(r.is_err());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn parse_fixed_array_header_invalid_version() {
|
fn parse_fixed_array_header_invalid_version() {
|
||||||
let mut buf = vec![0u8; 256];
|
let mut buf = vec![0u8; 256];
|
||||||
|
|||||||
@@ -245,8 +245,20 @@ impl Selection {
|
|||||||
|
|
||||||
match sel_type {
|
match sel_type {
|
||||||
// ALL / NONE: type(4) + version(4) + reserved(4) + length(4) = 16 bytes.
|
// ALL / NONE: type(4) + version(4) + reserved(4) + length(4) = 16 bytes.
|
||||||
3 => Ok((Selection::All, 16)),
|
3 | 0 => {
|
||||||
0 => Ok((Selection::None, 16)),
|
if data.len() < 16 {
|
||||||
|
return Err(FormatError::UnexpectedEof {
|
||||||
|
expected: 16,
|
||||||
|
available: data.len(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
let sel = if sel_type == 3 {
|
||||||
|
Selection::All
|
||||||
|
} else {
|
||||||
|
Selection::None
|
||||||
|
};
|
||||||
|
Ok((sel, 16))
|
||||||
|
}
|
||||||
2 => decode_hyperslab_serialized(data, version),
|
2 => decode_hyperslab_serialized(data, version),
|
||||||
1 => Err(FormatError::ChunkedReadError(
|
1 => Err(FormatError::ChunkedReadError(
|
||||||
"VDS point selections are not supported".into(),
|
"VDS point selections are not supported".into(),
|
||||||
@@ -274,12 +286,18 @@ impl Selection {
|
|||||||
/// selection. Hyperslab/point selections whose rank differs from
|
/// selection. Hyperslab/point selections whose rank differs from
|
||||||
/// `dims.len()` are rejected.
|
/// `dims.len()` are rejected.
|
||||||
pub fn iter_linear(&self, dims: &[u64]) -> Result<Vec<u64>, FormatError> {
|
pub fn iter_linear(&self, dims: &[u64]) -> Result<Vec<u64>, FormatError> {
|
||||||
let total: u64 = dims.iter().product();
|
let overflow = || FormatError::Overflow("VDS selection index overflow".into());
|
||||||
|
let total: u64 = dims
|
||||||
|
.iter()
|
||||||
|
.try_fold(1u64, |acc, &d| acc.checked_mul(d))
|
||||||
|
.ok_or_else(overflow)?;
|
||||||
// Row-major strides: row_stride[d] = product(dims[d+1..]).
|
// Row-major strides: row_stride[d] = product(dims[d+1..]).
|
||||||
let rank = dims.len();
|
let rank = dims.len();
|
||||||
let mut row_stride = vec![1u64; rank];
|
let mut row_stride = vec![1u64; rank];
|
||||||
for d in (0..rank.saturating_sub(1)).rev() {
|
for d in (0..rank.saturating_sub(1)).rev() {
|
||||||
row_stride[d] = row_stride[d + 1] * dims[d + 1];
|
row_stride[d] = row_stride[d + 1]
|
||||||
|
.checked_mul(dims[d + 1])
|
||||||
|
.ok_or_else(overflow)?;
|
||||||
}
|
}
|
||||||
|
|
||||||
match self {
|
match self {
|
||||||
@@ -301,9 +319,14 @@ impl Selection {
|
|||||||
for d in 0..rank {
|
for d in 0..rank {
|
||||||
let mut coords = Vec::new();
|
let mut coords = Vec::new();
|
||||||
for ci in 0..count[d] {
|
for ci in 0..count[d] {
|
||||||
let base = start[d] + ci * stride[d];
|
let base = ci
|
||||||
|
.checked_mul(stride[d])
|
||||||
|
.and_then(|o| start[d].checked_add(o))
|
||||||
|
.ok_or_else(overflow)?;
|
||||||
for bi in 0..block[d] {
|
for bi in 0..block[d] {
|
||||||
let coord = base + bi;
|
let coord = base.checked_add(bi).ok_or_else(overflow)?;
|
||||||
|
// Anything past the extent is malformed; bail before the
|
||||||
|
// coordinate list can grow without bound.
|
||||||
if coord >= dims[d] {
|
if coord >= dims[d] {
|
||||||
return Err(FormatError::ChunkedReadError(
|
return Err(FormatError::ChunkedReadError(
|
||||||
"VDS hyperslab selection exceeds dataspace extent".into(),
|
"VDS hyperslab selection exceeds dataspace extent".into(),
|
||||||
@@ -318,13 +341,19 @@ impl Selection {
|
|||||||
return Ok(Vec::new());
|
return Ok(Vec::new());
|
||||||
}
|
}
|
||||||
// Cartesian product in row-major order (dim 0 slowest-varying).
|
// Cartesian product in row-major order (dim 0 slowest-varying).
|
||||||
let out_len: usize = per_dim.iter().map(|c| c.len()).product();
|
let out_len: usize = per_dim
|
||||||
|
.iter()
|
||||||
|
.try_fold(1usize, |acc, c| acc.checked_mul(c.len()))
|
||||||
|
.ok_or_else(overflow)?;
|
||||||
let mut out = Vec::with_capacity(out_len);
|
let mut out = Vec::with_capacity(out_len);
|
||||||
let mut idx = vec![0usize; rank];
|
let mut idx = vec![0usize; rank];
|
||||||
loop {
|
loop {
|
||||||
let mut lin = 0u64;
|
let mut lin = 0u64;
|
||||||
for d in 0..rank {
|
for d in 0..rank {
|
||||||
lin += per_dim[d][idx[d]] * row_stride[d];
|
lin = per_dim[d][idx[d]]
|
||||||
|
.checked_mul(row_stride[d])
|
||||||
|
.and_then(|o| lin.checked_add(o))
|
||||||
|
.ok_or_else(overflow)?;
|
||||||
}
|
}
|
||||||
out.push(lin);
|
out.push(lin);
|
||||||
// Increment the mixed-radix counter, last dimension fastest.
|
// Increment the mixed-radix counter, last dimension fastest.
|
||||||
@@ -358,7 +387,10 @@ impl Selection {
|
|||||||
"VDS point selection exceeds dataspace extent".into(),
|
"VDS point selection exceeds dataspace extent".into(),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
lin += p[d] * row_stride[d];
|
lin = p[d]
|
||||||
|
.checked_mul(row_stride[d])
|
||||||
|
.and_then(|o| lin.checked_add(o))
|
||||||
|
.ok_or_else(overflow)?;
|
||||||
}
|
}
|
||||||
out.push(lin);
|
out.push(lin);
|
||||||
}
|
}
|
||||||
@@ -400,6 +432,13 @@ fn decode_hyperslab_serialized(
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
let rank = u32::from_le_bytes([data[10], data[11], data[12], data[13]]) as usize;
|
let rank = u32::from_le_bytes([data[10], data[11], data[12], data[13]]) as usize;
|
||||||
|
// HDF5 caps dataspace rank at 32 (H5S_MAX_RANK). Reject anything larger so a
|
||||||
|
// corrupt rank can't drive a huge allocation or read loop.
|
||||||
|
if rank > 32 {
|
||||||
|
return Err(FormatError::ChunkedReadError(
|
||||||
|
"hyperslab selection rank exceeds maximum (32)".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
let mut pos = 14;
|
let mut pos = 14;
|
||||||
let read_coord = |data: &[u8], pos: usize| -> Result<u64, FormatError> {
|
let read_coord = |data: &[u8], pos: usize| -> Result<u64, FormatError> {
|
||||||
if pos + enc_size > data.len() {
|
if pos + enc_size > data.len() {
|
||||||
@@ -660,4 +699,45 @@ mod tests {
|
|||||||
};
|
};
|
||||||
assert!(sel.iter_linear(&[4, 4]).is_err());
|
assert!(sel.iter_linear(&[4, 4]).is_err());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ----- Adversarial / hardening: malformed input must error, never panic -----
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn decode_all_truncated_does_not_overrun() {
|
||||||
|
// ALL claims to consume 16 bytes but only 8 are present.
|
||||||
|
let bytes = [3u8, 0, 0, 0, 1, 0, 0, 0];
|
||||||
|
assert!(Selection::decode_serialized(&bytes).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn decode_hyperslab_huge_rank_rejected() {
|
||||||
|
// rank = 0xFFFFFFFF must not drive a giant allocation.
|
||||||
|
let bytes = [
|
||||||
|
0x02u8, 0, 0, 0, 0x03, 0, 0, 0, 0x01, 0x02, 0xFF, 0xFF, 0xFF, 0xFF,
|
||||||
|
];
|
||||||
|
assert!(Selection::decode_serialized(&bytes).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn iter_linear_hyperslab_overflow_is_error() {
|
||||||
|
// start/stride/count near u64::MAX must not panic on multiply/add.
|
||||||
|
let sel = Selection::Hyperslab {
|
||||||
|
start: vec![u64::MAX - 1],
|
||||||
|
stride: vec![u64::MAX],
|
||||||
|
count: vec![u64::MAX],
|
||||||
|
block: vec![u64::MAX],
|
||||||
|
};
|
||||||
|
assert!(sel.iter_linear(&[100]).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn iter_linear_dims_product_overflow_is_error() {
|
||||||
|
assert!(Selection::All.iter_linear(&[u64::MAX, u64::MAX]).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn decode_empty_or_short_is_error_not_panic() {
|
||||||
|
assert!(Selection::decode_serialized(&[]).is_err());
|
||||||
|
assert!(Selection::decode_serialized(&[2, 0, 0, 0, 3, 0]).is_err());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Binary file not shown.
@@ -709,6 +709,36 @@ fn scaleoffset_float_escale_reads_as_raw() {
|
|||||||
assert_eq!(values, expect, "E-scale (raw + masked filter) must read verbatim");
|
assert_eq!(values, expect, "E-scale (raw + masked filter) must read verbatim");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn v4_virtual_dataset_cycle_errors_not_overflow() {
|
||||||
|
// virt -> virt2 -> virt (both virtual, same file). The reader must reject
|
||||||
|
// the nested virtual source rather than recurse into a stack overflow.
|
||||||
|
let file_data = include_bytes!("fixtures/vds_cyclic.h5");
|
||||||
|
let offset = find_signature(file_data).unwrap();
|
||||||
|
let sb = Superblock::parse(file_data, offset).unwrap();
|
||||||
|
let addr = resolve_path_any(file_data, &sb, "virt").unwrap();
|
||||||
|
let hdr = ObjectHeader::parse(file_data, addr as usize, sb.offset_size, sb.length_size).unwrap();
|
||||||
|
let ds = Dataspace::parse(
|
||||||
|
&hdr.messages.iter().find(|m| m.msg_type == MessageType::Dataspace).unwrap().data,
|
||||||
|
sb.length_size,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let (dt, _) = Datatype::parse(
|
||||||
|
&hdr.messages.iter().find(|m| m.msg_type == MessageType::Datatype).unwrap().data,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let layout = DataLayout::parse(
|
||||||
|
&hdr.messages.iter().find(|m| m.msg_type == MessageType::DataLayout).unwrap().data,
|
||||||
|
sb.offset_size,
|
||||||
|
sb.length_size,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let r = read_raw_data_full(
|
||||||
|
file_data, &layout, &ds, &dt, None, sb.offset_size, sb.length_size,
|
||||||
|
);
|
||||||
|
assert!(r.is_err(), "cyclic virtual dataset must error, not overflow");
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn v4_virtual_dataset_external_file_read() {
|
fn v4_virtual_dataset_external_file_read() {
|
||||||
use clawhdf5_format::data_read::read_raw_data_full_with_resolver;
|
use clawhdf5_format::data_read::read_raw_data_full_with_resolver;
|
||||||
|
|||||||
Reference in New Issue
Block a user