harden: make the new readers panic-free on malformed input

The readers added this cycle parse untrusted bytes, so malformed/hostile
input must produce errors — never a panic, OOM, or unbounded recursion.
Audited each new surface and fixed the concrete vectors, each covered by an
adversarial regression test:

- Paged Fixed Array: `1 << max_nelmts_bits` shift overflow (u8 up to 255);
  element-count bounded by file size; element/page offset multiplies checked.
- H5S selection decoder: ALL/NONE validate they have the 16 bytes they claim
  to consume; hyperslab rank capped at 32 (H5S_MAX_RANK); iter_linear
  coordinate/stride/product arithmetic uses checked ops.
- VDS mapping parser: drop pre-allocation from the untrusted `nused`;
  bounds-check all selection slicing.
- scale-offset / N-Bit filters: `1 << minbits` overflow at minbits==64; N-Bit
  `bit_offset + precision` overflow; N-Bit type-tree recursion depth capped to
  stop a crafted nested tree from overflowing the stack; element counts bounded
  by the chunk's expected decompressed size (threaded the previously-unused
  chunk_size into both decoders) so a bogus count can't over-allocate.
- VDS assembly: a virtual dataset whose source is itself virtual (a cycle) now
  errors instead of recursing into a stack overflow.

16 new adversarial tests; full format suite (482 lib) + agent + facade green;
clippy clean.

Co-Authored-By: Claude Opus 4.8 <[email protected]>
This commit is contained in:
osobh
2026-06-04 00:28:33 +00:00
co-authored by Claude Opus 4.8
parent 4fa7e89a46
commit e0189cd5c4
8 changed files with 417 additions and 48 deletions
+21
View File
@@ -2,6 +2,27 @@
## Unreleased ## Unreleased
### Robustness
- `clawhdf5-format`: harden the readers added this cycle against malformed /
hostile input — they parse untrusted bytes and must return errors, never
panic, OOM, or recurse without bound. Fixed concrete vectors found by audit
and locked in with adversarial tests:
- **Paged Fixed Array**: `1 << max_nelmts_bits` shift overflow (a `u8` ≥ 64);
element/page offset multiplications now checked; element count bounded by
file size.
- **H5S selection decoder**: `ALL`/`NONE` no longer claim 16 bytes they don't
have; hyperslab `rank` capped at 32 (`H5S_MAX_RANK`) to stop a giant
allocation; `iter_linear` coordinate/stride/product arithmetic is checked.
- **VDS mapping parser**: no pre-allocation from the untrusted `nused`; all
selection slicing is bounds-checked.
- **scale-offset / N-Bit filters**: `1 << minbits` overflow at `minbits == 64`;
N-Bit `bit_offset + precision` overflow; N-Bit type-tree recursion depth
capped (no stack overflow from a crafted nested tree); element counts
bounded by the chunk's expected decompressed size so a bogus count can't
drive a huge allocation.
- **Virtual Dataset assembly**: a virtual dataset whose source is itself
virtual (a cycle) now errors instead of recursing into a stack overflow.
### New Features ### New Features
- `clawhdf5-agent`: **compress fixed-length string datasets** (memory text - `clawhdf5-agent`: **compress fixed-length string datasets** (memory text
chunks, session summaries, ids, tags, entity/relation names, …). These were chunks, session summaries, ids, tags, entity/relation names, …). These were
+56 -10
View File
@@ -104,7 +104,29 @@ pub fn parse_vds_mappings(
let nused = read_length(heap_data, pos, length_size)?; let nused = read_length(heap_data, pos, length_size)?;
pos += ls; pos += ls;
let mut mappings = Vec::with_capacity(nused as usize); // `nused` is untrusted; don't pre-allocate from it. Each entry consumes at
// least a few bytes, so the loop is naturally bounded by the heap data and
// a bogus `nused` simply errors out on the first short read.
let mut mappings = Vec::new();
// Reads one self-describing selection at `pos`, returning its raw bytes and
// advancing past it — bounds-checked so a corrupt selection can't overrun.
let read_selection = |heap_data: &[u8], pos: &mut usize| -> Result<Vec<u8>, FormatError> {
let rest = heap_data.get(*pos..).ok_or(FormatError::UnexpectedEof {
expected: *pos,
available: heap_data.len(),
})?;
let (_, len) = Selection::decode_serialized(rest)?;
let bytes = rest
.get(..len)
.ok_or(FormatError::UnexpectedEof {
expected: pos.saturating_add(len),
available: heap_data.len(),
})?
.to_vec();
*pos += len;
Ok(bytes)
};
for _ in 0..nused { for _ in 0..nused {
// Source file name (with the version-1 same-file marker handled). // Source file name (with the version-1 same-file marker handled).
let source_file = if version >= 1 && heap_data.get(pos) == Some(&0x04) { let source_file = if version >= 1 && heap_data.get(pos) == Some(&0x04) {
@@ -117,15 +139,9 @@ pub fn parse_vds_mappings(
// Source dataset name. // Source dataset name.
let source_dataset = read_null_terminated_string(heap_data, &mut pos)?; let source_dataset = read_null_terminated_string(heap_data, &mut pos)?;
// Source selection (self-describing length). // Source selection, then virtual selection (both self-describing length).
let (_, ssel_len) = Selection::decode_serialized(&heap_data[pos..])?; let source_selection = read_selection(heap_data, &mut pos)?;
let source_selection = heap_data[pos..pos + ssel_len].to_vec(); let virtual_selection = read_selection(heap_data, &mut pos)?;
pos += ssel_len;
// Virtual selection.
let (_, vsel_len) = Selection::decode_serialized(&heap_data[pos..])?;
let virtual_selection = heap_data[pos..pos + vsel_len].to_vec();
pos += vsel_len;
mappings.push(VdsMapping { mappings.push(VdsMapping {
source_file, source_file,
@@ -815,4 +831,34 @@ mod tests {
assert_eq!(mappings[0].source_file, "src_ext.h5"); assert_eq!(mappings[0].source_file, "src_ext.h5");
assert_eq!(mappings[0].source_dataset, "data"); assert_eq!(mappings[0].source_dataset, "data");
} }
#[test]
fn parse_vds_mappings_huge_nused_does_not_oom_or_panic() {
// nused = u64::MAX with no entry data: must error, not pre-allocate or
// overrun.
let mut blob = vec![0x01u8];
blob.extend_from_slice(&u64::MAX.to_le_bytes());
assert!(parse_vds_mappings(&blob, 8).is_err());
}
#[test]
fn parse_vds_mappings_truncated_selection_does_not_overrun() {
// One entry whose source selection (ALL) is truncated to 8 of 16 bytes.
let blob = [
0x01u8, // version 1
0x01, 0, 0, 0, 0, 0, 0, 0, // nused = 1
0x04, // same-file marker
0x78, 0x00, // "x\0"
0x03, 0, 0, 0, 0x01, 0, 0, 0, // ALL header, truncated (8 of 16 bytes)
];
assert!(parse_vds_mappings(&blob, 8).is_err());
}
#[test]
fn parse_vds_mappings_empty_is_ok_empty() {
assert!(parse_vds_mappings(&[], 8).unwrap().is_empty());
// Header present, nused = 0.
let blob = [0x01u8, 0, 0, 0, 0, 0, 0, 0, 0];
assert!(parse_vds_mappings(&blob, 8).unwrap().is_empty());
}
} }
+8
View File
@@ -554,6 +554,14 @@ fn read_named_dataset_raw(
let dl_msg = find(MessageType::DataLayout) let dl_msg = find(MessageType::DataLayout)
.ok_or_else(|| FormatError::ChunkedReadError("VDS source has no data layout".into()))?; .ok_or_else(|| FormatError::ChunkedReadError("VDS source has no data layout".into()))?;
let layout = DataLayout::parse(&dl_msg.data, sb.offset_size, sb.length_size)?; let layout = DataLayout::parse(&dl_msg.data, sb.offset_size, sb.length_size)?;
// A virtual dataset whose source is itself another virtual dataset could
// form a cycle (A -> B -> A) and recurse into a stack overflow. Nested
// virtual sources are exotic and unsupported, so stop here cleanly.
if matches!(layout, DataLayout::Virtual { .. }) {
return Err(FormatError::ChunkedReadError(
"virtual dataset source is itself virtual (unsupported)".into(),
));
}
let pipeline = find(MessageType::FilterPipeline) let pipeline = find(MessageType::FilterPipeline)
.map(|m| FilterPipeline::parse(&m.data)) .map(|m| FilterPipeline::parse(&m.data))
.transpose()?; .transpose()?;
+139 -24
View File
@@ -17,7 +17,7 @@ use crate::filter_pipeline::{
pub fn decompress_chunk( pub fn decompress_chunk(
compressed: &[u8], compressed: &[u8],
pipeline: &FilterPipeline, pipeline: &FilterPipeline,
_chunk_size: usize, chunk_size: usize,
element_size: u32, element_size: u32,
) -> Result<Vec<u8>, FormatError> { ) -> Result<Vec<u8>, FormatError> {
let mut data = compressed.to_vec(); let mut data = compressed.to_vec();
@@ -29,8 +29,10 @@ pub fn decompress_chunk(
FILTER_LZ4 => lz4_decompress(&data)?, FILTER_LZ4 => lz4_decompress(&data)?,
FILTER_ZSTD => zstd_decompress(&data)?, FILTER_ZSTD => zstd_decompress(&data)?,
FILTER_FLETCHER32 => fletcher32_verify(&data)?, FILTER_FLETCHER32 => fletcher32_verify(&data)?,
FILTER_SCALEOFFSET => scaleoffset_decompress(&data, &filter.client_data)?, // `chunk_size` is the expected decompressed size; pass it so these
FILTER_NBIT => nbit_decompress(&data, &filter.client_data)?, // decoders can reject an element count that would over-allocate.
FILTER_SCALEOFFSET => scaleoffset_decompress(&data, &filter.client_data, chunk_size)?,
FILTER_NBIT => nbit_decompress(&data, &filter.client_data, chunk_size)?,
other => return Err(FormatError::UnsupportedFilter(other)), other => return Err(FormatError::UnsupportedFilter(other)),
}; };
} }
@@ -86,7 +88,11 @@ pub fn compress_chunk(
/// (0 = float D-scale, 2 = integer), `[1]`=scale factor (decimal digits for /// (0 = float D-scale, 2 = integer), `[1]`=scale factor (decimal digits for
/// D-scale), `[2]`=element count, `[4]`=element size, `[5]`=signed flag, /// D-scale), `[2]`=element count, `[4]`=element size, `[5]`=signed flag,
/// `[6]`=byte order (1 = big-endian), `[7]`=fill defined, `[8..]`=fill value. /// `[6]`=byte order (1 = big-endian), `[7]`=fill defined, `[8..]`=fill value.
fn scaleoffset_decompress(data: &[u8], cd: &[u32]) -> Result<Vec<u8>, FormatError> { fn scaleoffset_decompress(
data: &[u8],
cd: &[u32],
expected_bytes: usize,
) -> Result<Vec<u8>, FormatError> {
const H5Z_SO_FLOAT_DSCALE: u32 = 0; const H5Z_SO_FLOAT_DSCALE: u32 = 0;
const H5Z_SO_INT: u32 = 2; const H5Z_SO_INT: u32 = 2;
if cd.len() < 8 { if cd.len() < 8 {
@@ -107,6 +113,17 @@ fn scaleoffset_decompress(data: &[u8], cd: &[u32]) -> Result<Vec<u8>, FormatErro
"scale-offset: unsupported element size".into(), "scale-offset: unsupported element size".into(),
)); ));
} }
// The decoded output must match the chunk's uncompressed size; reject an
// element count that would over-allocate (e.g. minbits == 0 with a huge
// nelmts and no packed payload to bound it).
let out_bytes = nelmts
.checked_mul(elem_size)
.ok_or_else(|| FormatError::ChunkedReadError("scale-offset: size overflow".into()))?;
if expected_bytes != 0 && out_bytes > expected_bytes {
return Err(FormatError::ChunkedReadError(
"scale-offset: element count exceeds chunk size".into(),
));
}
let signed = cd[5] == 1; let signed = cd[5] == 1;
let big_endian = cd[6] == 1; let big_endian = cd[6] == 1;
let fill_defined = cd[7] == 1; let fill_defined = cd[7] == 1;
@@ -163,7 +180,14 @@ fn scaleoffset_decompress(data: &[u8], cd: &[u32]) -> Result<Vec<u8>, FormatErro
}; };
// The fill code (all ones) only exists when there are bits to pack. // The fill code (all ones) only exists when there are bits to pack.
let has_fill_code = fill_defined && minbits > 0 && minbits < 64; let has_fill_code = fill_defined && minbits > 0 && minbits < 64;
let fill_code: u64 = if minbits == 0 { 0 } else { (1u64 << minbits) - 1 }; // Computed for all 1..=64 widths; `1 << 64` would overflow, so saturate.
let fill_code: u64 = if minbits == 0 {
0
} else if minbits >= 64 {
u64::MAX
} else {
(1u64 << minbits) - 1
};
if is_float { if is_float {
let scale = 10f64.powi(cd[1] as i32); let scale = 10f64.powi(cd[1] as i32);
@@ -339,11 +363,20 @@ fn nbit_cd(cd: &[u32], i: usize) -> Result<u32, FormatError> {
.ok_or_else(|| FormatError::ChunkedReadError("nbit: truncated client data".into())) .ok_or_else(|| FormatError::ChunkedReadError("nbit: truncated client data".into()))
} }
/// Maximum N-Bit type-tree nesting depth. Real types nest only a few levels;
/// the bound stops a crafted tree from recursing into a stack overflow.
const NBIT_MAX_DEPTH: u32 = 64;
/// Parse one N-Bit type node from the client-data tree, advancing `idx`. /// Parse one N-Bit type node from the client-data tree, advancing `idx`.
fn parse_nbit_node(cd: &[u32], idx: &mut usize) -> Result<NbitNode, FormatError> { fn parse_nbit_node(cd: &[u32], idx: &mut usize, depth: u32) -> Result<NbitNode, FormatError> {
const ATOMIC: u32 = 1; const ATOMIC: u32 = 1;
const ARRAY: u32 = 2; const ARRAY: u32 = 2;
const COMPOUND: u32 = 3; const COMPOUND: u32 = 3;
if depth > NBIT_MAX_DEPTH {
return Err(FormatError::ChunkedReadError(
"nbit: type tree nested too deeply".into(),
));
}
let class = nbit_cd(cd, *idx)?; let class = nbit_cd(cd, *idx)?;
match class { match class {
ATOMIC => { ATOMIC => {
@@ -353,7 +386,11 @@ fn parse_nbit_node(cd: &[u32], idx: &mut usize) -> Result<NbitNode, FormatError>
let precision = nbit_cd(cd, *idx + 3)?; let precision = nbit_cd(cd, *idx + 3)?;
let bit_offset = nbit_cd(cd, *idx + 4)?; let bit_offset = nbit_cd(cd, *idx + 4)?;
*idx += 5; *idx += 5;
if size == 0 || size > 8 || precision == 0 || bit_offset + precision > (size * 8) as u32 let end_bit = bit_offset.checked_add(precision);
if size == 0
|| size > 8
|| precision == 0
|| end_bit.is_none_or(|e| e > (size * 8) as u32)
{ {
return Err(FormatError::ChunkedReadError( return Err(FormatError::ChunkedReadError(
"nbit: invalid atomic parameters".into(), "nbit: invalid atomic parameters".into(),
@@ -370,7 +407,7 @@ fn parse_nbit_node(cd: &[u32], idx: &mut usize) -> Result<NbitNode, FormatError>
// class, total size, base type node // class, total size, base type node
let total = nbit_cd(cd, *idx + 1)? as usize; let total = nbit_cd(cd, *idx + 1)? as usize;
*idx += 2; *idx += 2;
let base = parse_nbit_node(cd, idx)?; let base = parse_nbit_node(cd, idx, depth + 1)?;
let base_size = base.byte_size(); let base_size = base.byte_size();
if base_size == 0 || !total.is_multiple_of(base_size) { if base_size == 0 || !total.is_multiple_of(base_size) {
return Err(FormatError::ChunkedReadError( return Err(FormatError::ChunkedReadError(
@@ -388,12 +425,17 @@ fn parse_nbit_node(cd: &[u32], idx: &mut usize) -> Result<NbitNode, FormatError>
let total = nbit_cd(cd, *idx + 1)? as usize; let total = nbit_cd(cd, *idx + 1)? as usize;
let nmembers = nbit_cd(cd, *idx + 2)? as usize; let nmembers = nbit_cd(cd, *idx + 2)? as usize;
*idx += 3; *idx += 3;
let mut members = Vec::with_capacity(nmembers); // Don't pre-allocate from the untrusted member count; the loop is
// bounded by `nbit_cd` running out of client data.
let mut members = Vec::new();
for _ in 0..nmembers { for _ in 0..nmembers {
let moff = nbit_cd(cd, *idx)? as usize; let moff = nbit_cd(cd, *idx)? as usize;
*idx += 1; *idx += 1;
let node = parse_nbit_node(cd, idx)?; let node = parse_nbit_node(cd, idx, depth + 1)?;
if moff + node.byte_size() > total { if moff
.checked_add(node.byte_size())
.is_none_or(|end| end > total)
{
return Err(FormatError::ChunkedReadError( return Err(FormatError::ChunkedReadError(
"nbit: member exceeds compound size".into(), "nbit: member exceeds compound size".into(),
)); ));
@@ -488,7 +530,7 @@ fn decode_nbit_node(
/// (HDF5's canonical reduced-precision layout). Sign-extension of reduced /// (HDF5's canonical reduced-precision layout). Sign-extension of reduced
/// precision signed integers is the datatype reader's job. Atomic floats are /// precision signed integers is the datatype reader's job. Atomic floats are
/// encoded as full-precision atomics and handled transparently. /// encoded as full-precision atomics and handled transparently.
fn nbit_decompress(data: &[u8], cd: &[u32]) -> Result<Vec<u8>, FormatError> { fn nbit_decompress(data: &[u8], cd: &[u32], expected_bytes: usize) -> Result<Vec<u8>, FormatError> {
if cd.len() < 4 { if cd.len() < 4 {
return Err(FormatError::ChunkedReadError( return Err(FormatError::ChunkedReadError(
"nbit: missing filter client data".into(), "nbit: missing filter client data".into(),
@@ -496,7 +538,7 @@ fn nbit_decompress(data: &[u8], cd: &[u32]) -> Result<Vec<u8>, FormatError> {
} }
let nelmts = cd[2] as usize; let nelmts = cd[2] as usize;
let mut idx = 3; let mut idx = 3;
let root = parse_nbit_node(cd, &mut idx)?; let root = parse_nbit_node(cd, &mut idx, 0)?;
let elem_size = root.byte_size(); let elem_size = root.byte_size();
if elem_size == 0 { if elem_size == 0 {
return Err(FormatError::ChunkedReadError( return Err(FormatError::ChunkedReadError(
@@ -507,6 +549,13 @@ fn nbit_decompress(data: &[u8], cd: &[u32]) -> Result<Vec<u8>, FormatError> {
let total = nelmts let total = nelmts
.checked_mul(elem_size) .checked_mul(elem_size)
.ok_or_else(|| FormatError::ChunkedReadError("nbit: size overflow".into()))?; .ok_or_else(|| FormatError::ChunkedReadError("nbit: size overflow".into()))?;
// The decoded output must match the chunk's uncompressed size; reject a
// count that would over-allocate.
if expected_bytes != 0 && total > expected_bytes {
return Err(FormatError::ChunkedReadError(
"nbit: element count exceeds chunk size".into(),
));
}
let mut out = vec![0u8; total]; let mut out = vec![0u8; total];
let mut br = BitReader { data, pos: 0 }; let mut br = BitReader { data, pos: 0 };
for elem in out.chunks_exact_mut(elem_size) { for elem in out.chunks_exact_mut(elem_size) {
@@ -1176,7 +1225,7 @@ mod tests {
0x02, 0x00, 0x00, 0x00, 0x08, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x08, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc6, 0x00,
]; ];
assert_eq!(scaleoffset_decompress(&raw, &cd).unwrap(), i32_le(&[0, 1, 2, 3])); assert_eq!(scaleoffset_decompress(&raw, &cd, 0).unwrap(), i32_le(&[0, 1, 2, 3]));
} }
#[test] #[test]
@@ -1188,7 +1237,7 @@ mod tests {
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x4f, 0x79, 0xce, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x4f, 0x79, 0xce, 0x00,
]; ];
assert_eq!( assert_eq!(
scaleoffset_decompress(&raw, &cd).unwrap(), scaleoffset_decompress(&raw, &cd, 0).unwrap(),
i32_le(&[-5, -3, -1, 0, 2, 4, 7, 9]) i32_le(&[-5, -3, -1, 0, 2, 4, 7, 9])
); );
} }
@@ -1202,7 +1251,7 @@ mod tests {
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x23, 0x45, 0x67, 0x89, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x23, 0x45, 0x67, 0x89, 0x00,
]; ];
let expected: Vec<u8> = (100u32..110).flat_map(|v| v.to_le_bytes()).collect(); let expected: Vec<u8> = (100u32..110).flat_map(|v| v.to_le_bytes()).collect();
assert_eq!(scaleoffset_decompress(&raw, &cd).unwrap(), expected); assert_eq!(scaleoffset_decompress(&raw, &cd, 0).unwrap(), expected);
} }
fn as_f32(bytes: &[u8]) -> Vec<f32> { fn as_f32(bytes: &[u8]) -> Vec<f32> {
@@ -1220,7 +1269,7 @@ mod tests {
0x05, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x80, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x80, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x15, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x15, 0x40,
]; ];
let got = as_f32(&scaleoffset_decompress(&raw, &cd).unwrap()); let got = as_f32(&scaleoffset_decompress(&raw, &cd, 0).unwrap());
assert_eq!(got, vec![0.0, 1.0, 2.0, 3.0]); assert_eq!(got, vec![0.0, 1.0, 2.0, 3.0]);
} }
@@ -1232,7 +1281,7 @@ mod tests {
0x08, 0x00, 0x00, 0x00, 0x08, 0xcd, 0xcc, 0xcc, 0x3d, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x08, 0xcd, 0xcc, 0xcc, 0x3d, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0x00, 0x64, 0xc8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0x00, 0x64, 0xc8, 0x00,
]; ];
let got = as_f32(&scaleoffset_decompress(&raw, &cd).unwrap()); let got = as_f32(&scaleoffset_decompress(&raw, &cd, 0).unwrap());
let exp = [0.0f32, 0.1, 0.2, 0.3]; let exp = [0.0f32, 0.1, 0.2, 0.3];
assert_eq!(got.len(), 4); assert_eq!(got.len(), 4);
for (g, e) in got.iter().zip(exp.iter()) { for (g, e) in got.iter().zip(exp.iter()) {
@@ -1246,7 +1295,7 @@ mod tests {
let cd = [1u32, 3, 50, 1, 4, 0, 0, 1, 0]; let cd = [1u32, 3, 50, 1, 4, 0, 0, 1, 0];
let raw = [0u8; 24]; let raw = [0u8; 24];
assert!(matches!( assert!(matches!(
scaleoffset_decompress(&raw, &cd), scaleoffset_decompress(&raw, &cd, 0),
Err(FormatError::UnsupportedFilter(FILTER_SCALEOFFSET)) Err(FormatError::UnsupportedFilter(FILTER_SCALEOFFSET))
)); ));
} }
@@ -1265,7 +1314,7 @@ mod tests {
.iter() .iter()
.flat_map(|v| v.to_le_bytes()) .flat_map(|v| v.to_le_bytes())
.collect(); .collect();
assert_eq!(nbit_decompress(&raw, &cd).unwrap(), expected); assert_eq!(nbit_decompress(&raw, &cd, 0).unwrap(), expected);
} }
#[test] #[test]
@@ -1282,7 +1331,7 @@ mod tests {
0x64, 0x00, 0x00, 0x00, // 0x00000064 0x64, 0x00, 0x00, 0x00, // 0x00000064
0x18, 0xfc, 0x00, 0x00, // 0x0000fc18 0x18, 0xfc, 0x00, 0x00, // 0x0000fc18
]; ];
assert_eq!(nbit_decompress(&raw, &cd).unwrap(), expected); assert_eq!(nbit_decompress(&raw, &cd, 0).unwrap(), expected);
} }
#[test] #[test]
@@ -1297,7 +1346,7 @@ mod tests {
0xe8,0x03,0x00,0x00, 0x07,0x00,0x00,0x00, // (1000, 7) 0xe8,0x03,0x00,0x00, 0x07,0x00,0x00,0x00, // (1000, 7)
0x00,0x80,0x00,0x00, 0xff,0x00,0x00,0x00, // (-32768, 255) 0x00,0x80,0x00,0x00, 0xff,0x00,0x00,0x00, // (-32768, 255)
]; ];
assert_eq!(nbit_decompress(&raw, &cd).unwrap(), expected); assert_eq!(nbit_decompress(&raw, &cd, 0).unwrap(), expected);
} }
#[test] #[test]
@@ -1311,7 +1360,7 @@ mod tests {
0xff,0xff,0x00,0x00, 0x64,0x00,0x00,0x00, 0xc8,0x00,0x00,0x00, // ([-1,100], 200) 0xff,0xff,0x00,0x00, 0x64,0x00,0x00,0x00, 0xc8,0x00,0x00,0x00, // ([-1,100], 200)
0xe8,0x03,0x00,0x00, 0x00,0x80,0x00,0x00, 0x07,0x00,0x00,0x00, // ([1000,-32768], 7) 0xe8,0x03,0x00,0x00, 0x00,0x80,0x00,0x00, 0x07,0x00,0x00,0x00, // ([1000,-32768], 7)
]; ];
assert_eq!(nbit_decompress(&raw, &cd).unwrap(), expected); assert_eq!(nbit_decompress(&raw, &cd, 0).unwrap(), expected);
} }
#[test] #[test]
@@ -1328,6 +1377,72 @@ mod tests {
0xff,0xff,0x00,0x00, 0x00,0x00,0xc0,0x3f, // (-1, 1.5) 0xff,0xff,0x00,0x00, 0x00,0x00,0xc0,0x3f, // (-1, 1.5)
0x64,0x00,0x00,0x00, 0x00,0x00,0x20,0xc0, // (100, -2.5) 0x64,0x00,0x00,0x00, 0x00,0x00,0x20,0xc0, // (100, -2.5)
]; ];
assert_eq!(nbit_decompress(&raw, &cd).unwrap(), expected); assert_eq!(nbit_decompress(&raw, &cd, 0).unwrap(), expected);
}
// ----- Adversarial / hardening: malformed filter data must not panic -----
#[test]
fn scaleoffset_minbits_64_does_not_panic() {
// minbits == 64 previously overflowed `1u64 << minbits` computing the
// fill code. cd: int, nelmts=1, elem_size=8, fill_defined=1.
let cd = [2u32, 0, 1, 0, 8, 1, 0, 1];
let mut data = Vec::new();
data.extend_from_slice(&64u32.to_le_bytes()); // minbits = 64
data.push(8); // minval_width
data.extend_from_slice(&0i64.to_le_bytes()); // minval
data.extend_from_slice(&[0u8; 8]); // reserved
data.extend_from_slice(&[0u8; 8]); // one 64-bit code
// Must return a Result (Ok or Err) without panicking.
let _ = scaleoffset_decompress(&data, &cd, 8);
}
#[test]
fn scaleoffset_huge_nelmts_bounded_by_chunk_size() {
// minbits == 0 (no packed payload) with a giant nelmts must not try to
// allocate when the expected chunk size is small.
let cd = [2u32, 0, u32::MAX, 0, 4, 0, 0, 0];
let mut data = Vec::new();
data.extend_from_slice(&0u32.to_le_bytes()); // minbits = 0
data.push(4);
data.extend_from_slice(&[0u8; 4]);
assert!(scaleoffset_decompress(&data, &cd, 64).is_err());
}
#[test]
fn nbit_deeply_nested_array_is_rejected() {
// A chain of ARRAY nodes (class 2) far deeper than NBIT_MAX_DEPTH must
// error rather than recurse into a stack overflow. Layout per level:
// [class=2, total]. Terminate with a (never-reached) atomic.
let mut cd = vec![0u32, 0, 1]; // nparms, flag, nelmts
for _ in 0..500 {
cd.push(2); // ARRAY
cd.push(8); // total size
}
cd.extend_from_slice(&[1, 8, 0, 8, 0]); // atomic leaf
assert!(nbit_decompress(&[], &cd, 0).is_err());
}
#[test]
fn nbit_atomic_bit_offset_overflow_is_rejected() {
// bit_offset + precision near u32::MAX must not overflow the check.
let cd = [0u32, 0, 1, 1, 8, 0, u32::MAX, u32::MAX];
assert!(nbit_decompress(&[0u8; 8], &cd, 0).is_err());
}
#[test]
fn nbit_huge_nelmts_bounded_by_chunk_size() {
// Valid 1-byte atomic but an enormous element count; the expected chunk
// size bounds the allocation.
let cd = [0u32, 0, u32::MAX, 1, 1, 0, 8, 0];
assert!(nbit_decompress(&[0u8; 4], &cd, 16).is_err());
}
#[test]
fn scaleoffset_truncated_inputs_do_not_panic() {
assert!(scaleoffset_decompress(&[], &[2, 0, 1, 0, 4, 0, 0, 0], 4).is_err());
assert!(scaleoffset_decompress(&[0u8; 3], &[2, 0, 1, 0, 4, 0, 0, 0], 4).is_err());
// Missing client data entirely.
assert!(scaleoffset_decompress(&[0u8; 32], &[2, 0], 4).is_err());
} }
} }
+74 -5
View File
@@ -144,12 +144,30 @@ pub fn read_fixed_array_chunks(
let elements_start = db_offset + db_header_size; let elements_start = db_offset + db_header_size;
let num_elements = header.num_elements as usize; let num_elements = header.num_elements as usize;
// A chunk index cannot describe more elements than the file has bytes (each
// element occupies at least `offset_size` bytes). Reject a corrupt count
// before it can drive a huge loop or overflow an offset computation.
if num_elements > file_data.len() {
return Err(FormatError::ChunkedReadError(
"Fixed Array element count exceeds file size".into(),
));
}
let os = offset_size as usize; let os = offset_size as usize;
// On-disk stride of one element. For non-filtered arrays the element is just // On-disk stride of one element. For non-filtered arrays the element is just
// the chunk address (== offset_size); for filtered arrays it is // the chunk address (== offset_size); for filtered arrays it is
// address + chunk_size + filter_mask (== header.element_size). // address + chunk_size + filter_mask (== header.element_size).
let elem_stride = (header.element_size as usize).max(os); let elem_stride = (header.element_size as usize).max(os);
// Absolute file offset of element `idx` within a run starting at `base`,
// with overflow surfaced as a clean error rather than a panic/wrap.
let elem_at = |base: usize, idx: usize| -> Result<usize, FormatError> {
idx.checked_mul(elem_stride)
.and_then(|o| base.checked_add(o))
.ok_or(FormatError::ChunkedReadError(
"Fixed Array element offset overflow".into(),
))
};
// Compute chunk offsets based on index. // Compute chunk offsets based on index.
// Chunks are stored in row-major order within the dataset space. // Chunks are stored in row-major order within the dataset space.
let mut num_chunks_per_dim = Vec::with_capacity(rank); let mut num_chunks_per_dim = Vec::with_capacity(rank);
@@ -189,6 +207,13 @@ pub fn read_fixed_array_chunks(
}; };
// A data block is paged when it holds more elements than fit in one page. // A data block is paged when it holds more elements than fit in one page.
// `max_nelmts_bits` is an untrusted u8; a shift >= the pointer width would
// panic, so reject it (real page-size bits are tiny — 10 by default).
if header.max_nelmts_bits as u32 >= usize::BITS {
return Err(FormatError::ChunkedReadError(
"Fixed Array max_nelmts_bits too large".into(),
));
}
let page_nelmts = 1usize << header.max_nelmts_bits; let page_nelmts = 1usize << header.max_nelmts_bits;
let is_paged = num_elements > page_nelmts; let is_paged = num_elements > page_nelmts;
@@ -196,7 +221,7 @@ pub fn read_fixed_array_chunks(
// Non-paged: prefix, then `num_elements` elements packed directly, // Non-paged: prefix, then `num_elements` elements packed directly,
// then a trailing checksum (which we don't validate). // then a trailing checksum (which we don't validate).
for i in 0..num_elements { for i in 0..num_elements {
push_element(i, elements_start + i * elem_stride, &mut chunks)?; push_element(i, elem_at(elements_start, i)?, &mut chunks)?;
} }
return Ok(chunks); return Ok(chunks);
} }
@@ -207,12 +232,18 @@ pub fn read_fixed_array_chunks(
// only the final page holds fewer elements. Uninitialized pages (bit clear) // only the final page holds fewer elements. Uninitialized pages (bit clear)
// still occupy their slot on disk but are zero-filled, so the bitmap — not a // still occupy their slot on disk but are zero-filled, so the bitmap — not a
// 0xFF sentinel — is what marks a whole page as unallocated. // 0xFF sentinel — is what marks a whole page as unallocated.
let stride_overflow = || {
FormatError::ChunkedReadError("Fixed Array page offset overflow".into())
};
let npages = num_elements.div_ceil(page_nelmts); let npages = num_elements.div_ceil(page_nelmts);
let bitmap_size = npages.div_ceil(8); let bitmap_size = npages.div_ceil(8);
let bitmap_start = elements_start; let bitmap_start = elements_start;
// prefix(db_header_size) + bitmap + checksum(4) // prefix(db_header_size) + bitmap + checksum(4)
let pages_start = db_offset + db_header_size + bitmap_size + 4; let pages_start = db_offset + db_header_size + bitmap_size + 4;
let page_stride = page_nelmts * elem_stride + 4; let page_stride = page_nelmts
.checked_mul(elem_stride)
.and_then(|x| x.checked_add(4))
.ok_or_else(stride_overflow)?;
if bitmap_start + bitmap_size > file_data.len() { if bitmap_start + bitmap_size > file_data.len() {
return Err(FormatError::UnexpectedEof { return Err(FormatError::UnexpectedEof {
@@ -222,7 +253,7 @@ pub fn read_fixed_array_chunks(
} }
for p in 0..npages { for p in 0..npages {
let page_first = p * page_nelmts; let page_first = p * page_nelmts; // < num_elements, cannot overflow
let page_count = core::cmp::min(page_nelmts, num_elements - page_first); let page_count = core::cmp::min(page_nelmts, num_elements - page_first);
// Check the page-init bit (MSB-first within each byte). // Check the page-init bit (MSB-first within each byte).
@@ -232,9 +263,12 @@ pub fn read_fixed_array_chunks(
continue; // entire page unallocated continue; // entire page unallocated
} }
let page_off = pages_start + p * page_stride; let page_off = p
.checked_mul(page_stride)
.and_then(|o| pages_start.checked_add(o))
.ok_or_else(stride_overflow)?;
for e in 0..page_count { for e in 0..page_count {
push_element(page_first + e, page_off + e * elem_stride, &mut chunks)?; push_element(page_first + e, elem_at(page_off, e)?, &mut chunks)?;
} }
} }
@@ -420,6 +454,41 @@ mod tests {
assert!(result.is_err()); assert!(result.is_err());
} }
/// Malformed headers must error, never panic (shift overflow, huge counts).
#[test]
fn read_rejects_oversized_max_nelmts_bits() {
let mut buf = vec![0u8; 512];
let fahd = 0x40usize;
buf[fahd..fahd + 4].copy_from_slice(b"FAHD");
buf[fahd + 4] = 0; // version
buf[fahd + 5] = 0; // client_id
buf[fahd + 6] = 8; // element_size
buf[fahd + 7] = 200; // max_nelmts_bits — absurd, would overflow a shift
buf[fahd + 8..fahd + 16].copy_from_slice(&3u64.to_le_bytes()); // num_elements
buf[fahd + 16..fahd + 24].copy_from_slice(&0x100u64.to_le_bytes());
// FADB so parsing reaches the paged check
let db = 0x100usize;
buf[db..db + 4].copy_from_slice(b"FADB");
let header = FixedArrayHeader::parse(&buf, fahd, 8, 8).unwrap();
let r = read_fixed_array_chunks(&buf, &header, &[100], &[20], 8, 8, 8);
assert!(r.is_err());
}
#[test]
fn read_rejects_num_elements_larger_than_file() {
let mut buf = vec![0u8; 256];
let fahd = 0x40usize;
buf[fahd..fahd + 4].copy_from_slice(b"FAHD");
buf[fahd + 6] = 8;
buf[fahd + 7] = 10;
buf[fahd + 8..fahd + 16].copy_from_slice(&u64::MAX.to_le_bytes()); // absurd count
buf[fahd + 16..fahd + 24].copy_from_slice(&0x80u64.to_le_bytes());
buf[0x80..0x84].copy_from_slice(b"FADB");
let header = FixedArrayHeader::parse(&buf, fahd, 8, 8).unwrap();
let r = read_fixed_array_chunks(&buf, &header, &[100], &[20], 8, 8, 8);
assert!(r.is_err());
}
#[test] #[test]
fn parse_fixed_array_header_invalid_version() { fn parse_fixed_array_header_invalid_version() {
let mut buf = vec![0u8; 256]; let mut buf = vec![0u8; 256];
+89 -9
View File
@@ -245,8 +245,20 @@ impl Selection {
match sel_type { match sel_type {
// ALL / NONE: type(4) + version(4) + reserved(4) + length(4) = 16 bytes. // ALL / NONE: type(4) + version(4) + reserved(4) + length(4) = 16 bytes.
3 => Ok((Selection::All, 16)), 3 | 0 => {
0 => Ok((Selection::None, 16)), if data.len() < 16 {
return Err(FormatError::UnexpectedEof {
expected: 16,
available: data.len(),
});
}
let sel = if sel_type == 3 {
Selection::All
} else {
Selection::None
};
Ok((sel, 16))
}
2 => decode_hyperslab_serialized(data, version), 2 => decode_hyperslab_serialized(data, version),
1 => Err(FormatError::ChunkedReadError( 1 => Err(FormatError::ChunkedReadError(
"VDS point selections are not supported".into(), "VDS point selections are not supported".into(),
@@ -274,12 +286,18 @@ impl Selection {
/// selection. Hyperslab/point selections whose rank differs from /// selection. Hyperslab/point selections whose rank differs from
/// `dims.len()` are rejected. /// `dims.len()` are rejected.
pub fn iter_linear(&self, dims: &[u64]) -> Result<Vec<u64>, FormatError> { pub fn iter_linear(&self, dims: &[u64]) -> Result<Vec<u64>, FormatError> {
let total: u64 = dims.iter().product(); let overflow = || FormatError::Overflow("VDS selection index overflow".into());
let total: u64 = dims
.iter()
.try_fold(1u64, |acc, &d| acc.checked_mul(d))
.ok_or_else(overflow)?;
// Row-major strides: row_stride[d] = product(dims[d+1..]). // Row-major strides: row_stride[d] = product(dims[d+1..]).
let rank = dims.len(); let rank = dims.len();
let mut row_stride = vec![1u64; rank]; let mut row_stride = vec![1u64; rank];
for d in (0..rank.saturating_sub(1)).rev() { for d in (0..rank.saturating_sub(1)).rev() {
row_stride[d] = row_stride[d + 1] * dims[d + 1]; row_stride[d] = row_stride[d + 1]
.checked_mul(dims[d + 1])
.ok_or_else(overflow)?;
} }
match self { match self {
@@ -301,9 +319,14 @@ impl Selection {
for d in 0..rank { for d in 0..rank {
let mut coords = Vec::new(); let mut coords = Vec::new();
for ci in 0..count[d] { for ci in 0..count[d] {
let base = start[d] + ci * stride[d]; let base = ci
.checked_mul(stride[d])
.and_then(|o| start[d].checked_add(o))
.ok_or_else(overflow)?;
for bi in 0..block[d] { for bi in 0..block[d] {
let coord = base + bi; let coord = base.checked_add(bi).ok_or_else(overflow)?;
// Anything past the extent is malformed; bail before the
// coordinate list can grow without bound.
if coord >= dims[d] { if coord >= dims[d] {
return Err(FormatError::ChunkedReadError( return Err(FormatError::ChunkedReadError(
"VDS hyperslab selection exceeds dataspace extent".into(), "VDS hyperslab selection exceeds dataspace extent".into(),
@@ -318,13 +341,19 @@ impl Selection {
return Ok(Vec::new()); return Ok(Vec::new());
} }
// Cartesian product in row-major order (dim 0 slowest-varying). // Cartesian product in row-major order (dim 0 slowest-varying).
let out_len: usize = per_dim.iter().map(|c| c.len()).product(); let out_len: usize = per_dim
.iter()
.try_fold(1usize, |acc, c| acc.checked_mul(c.len()))
.ok_or_else(overflow)?;
let mut out = Vec::with_capacity(out_len); let mut out = Vec::with_capacity(out_len);
let mut idx = vec![0usize; rank]; let mut idx = vec![0usize; rank];
loop { loop {
let mut lin = 0u64; let mut lin = 0u64;
for d in 0..rank { for d in 0..rank {
lin += per_dim[d][idx[d]] * row_stride[d]; lin = per_dim[d][idx[d]]
.checked_mul(row_stride[d])
.and_then(|o| lin.checked_add(o))
.ok_or_else(overflow)?;
} }
out.push(lin); out.push(lin);
// Increment the mixed-radix counter, last dimension fastest. // Increment the mixed-radix counter, last dimension fastest.
@@ -358,7 +387,10 @@ impl Selection {
"VDS point selection exceeds dataspace extent".into(), "VDS point selection exceeds dataspace extent".into(),
)); ));
} }
lin += p[d] * row_stride[d]; lin = p[d]
.checked_mul(row_stride[d])
.and_then(|o| lin.checked_add(o))
.ok_or_else(overflow)?;
} }
out.push(lin); out.push(lin);
} }
@@ -400,6 +432,13 @@ fn decode_hyperslab_serialized(
)); ));
} }
let rank = u32::from_le_bytes([data[10], data[11], data[12], data[13]]) as usize; let rank = u32::from_le_bytes([data[10], data[11], data[12], data[13]]) as usize;
// HDF5 caps dataspace rank at 32 (H5S_MAX_RANK). Reject anything larger so a
// corrupt rank can't drive a huge allocation or read loop.
if rank > 32 {
return Err(FormatError::ChunkedReadError(
"hyperslab selection rank exceeds maximum (32)".into(),
));
}
let mut pos = 14; let mut pos = 14;
let read_coord = |data: &[u8], pos: usize| -> Result<u64, FormatError> { let read_coord = |data: &[u8], pos: usize| -> Result<u64, FormatError> {
if pos + enc_size > data.len() { if pos + enc_size > data.len() {
@@ -660,4 +699,45 @@ mod tests {
}; };
assert!(sel.iter_linear(&[4, 4]).is_err()); assert!(sel.iter_linear(&[4, 4]).is_err());
} }
// ----- Adversarial / hardening: malformed input must error, never panic -----
#[test]
fn decode_all_truncated_does_not_overrun() {
// ALL claims to consume 16 bytes but only 8 are present.
let bytes = [3u8, 0, 0, 0, 1, 0, 0, 0];
assert!(Selection::decode_serialized(&bytes).is_err());
}
#[test]
fn decode_hyperslab_huge_rank_rejected() {
// rank = 0xFFFFFFFF must not drive a giant allocation.
let bytes = [
0x02u8, 0, 0, 0, 0x03, 0, 0, 0, 0x01, 0x02, 0xFF, 0xFF, 0xFF, 0xFF,
];
assert!(Selection::decode_serialized(&bytes).is_err());
}
#[test]
fn iter_linear_hyperslab_overflow_is_error() {
// start/stride/count near u64::MAX must not panic on multiply/add.
let sel = Selection::Hyperslab {
start: vec![u64::MAX - 1],
stride: vec![u64::MAX],
count: vec![u64::MAX],
block: vec![u64::MAX],
};
assert!(sel.iter_linear(&[100]).is_err());
}
#[test]
fn iter_linear_dims_product_overflow_is_error() {
assert!(Selection::All.iter_linear(&[u64::MAX, u64::MAX]).is_err());
}
#[test]
fn decode_empty_or_short_is_error_not_panic() {
assert!(Selection::decode_serialized(&[]).is_err());
assert!(Selection::decode_serialized(&[2, 0, 0, 0, 3, 0]).is_err());
}
} }
Binary file not shown.
@@ -709,6 +709,36 @@ fn scaleoffset_float_escale_reads_as_raw() {
assert_eq!(values, expect, "E-scale (raw + masked filter) must read verbatim"); assert_eq!(values, expect, "E-scale (raw + masked filter) must read verbatim");
} }
#[test]
fn v4_virtual_dataset_cycle_errors_not_overflow() {
// virt -> virt2 -> virt (both virtual, same file). The reader must reject
// the nested virtual source rather than recurse into a stack overflow.
let file_data = include_bytes!("fixtures/vds_cyclic.h5");
let offset = find_signature(file_data).unwrap();
let sb = Superblock::parse(file_data, offset).unwrap();
let addr = resolve_path_any(file_data, &sb, "virt").unwrap();
let hdr = ObjectHeader::parse(file_data, addr as usize, sb.offset_size, sb.length_size).unwrap();
let ds = Dataspace::parse(
&hdr.messages.iter().find(|m| m.msg_type == MessageType::Dataspace).unwrap().data,
sb.length_size,
)
.unwrap();
let (dt, _) = Datatype::parse(
&hdr.messages.iter().find(|m| m.msg_type == MessageType::Datatype).unwrap().data,
)
.unwrap();
let layout = DataLayout::parse(
&hdr.messages.iter().find(|m| m.msg_type == MessageType::DataLayout).unwrap().data,
sb.offset_size,
sb.length_size,
)
.unwrap();
let r = read_raw_data_full(
file_data, &layout, &ds, &dt, None, sb.offset_size, sb.length_size,
);
assert!(r.is_err(), "cyclic virtual dataset must error, not overflow");
}
#[test] #[test]
fn v4_virtual_dataset_external_file_read() { fn v4_virtual_dataset_external_file_read() {
use clawhdf5_format::data_read::read_raw_data_full_with_resolver; use clawhdf5_format::data_read::read_raw_data_full_with_resolver;