osobhandClaude Opus 5.5
d345ffbf80
h5rs (dump, ls, diff, check --data) kept its own lenient VL decoder: a heap object longer than its element was cut to the element's length (libhdf5 and h5py refuse it), a null string printed "" where h5dump prints NULL, the stored element size was trusted, and every heap collection was kept as an owned copy for the whole run. It now resolves each element with VlResolver::element / string_element (new: one element in place, borrowing from the file), and refuses a VL type whose stored element size is not 4 + offset size + 4, as File does. H5::heap_object and its cache are gone. h5diff compares a null VL string equal to an empty one; so does h5rs diff. clawhdf5-wasm already resolved VL strings with read_vl_strings; it now uses VlResolver and checks the stored element size before reading, as File::read_string does. Tests (h5py writes the files, patched for "a\0b", a null element and mis-sized heap objects, with 8- and 4-byte offsets): - h5rs_interop dump_prints_vl_data_like_h5dump: byte-identical to h5dump; - dump_json_vl_values_match_h5py: h5py's values, errors where h5py fails; - check_data_flags_mis_sized_vl_heap_objects; - clawhdf5-wasm tests/vl_strings.rs: wasm, File and h5py agree. All four fail before. check --data over the 150 cve_hdf5 CVE and fuzzer files now passes 15 (h5dump rejects 8 of them), was 16 and 9: the stored-size check flags cve-2024-32608. h5rs-check-ok-files.sh --data: 0 of 422 flagged; h5rs-fuzz.sh: clean on 180 files. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
472 lines
16 KiB
Rust
472 lines
16 KiB
Rust
//! Decoding one element of any datatype into a [`Value`], and printing it.
|
||
//!
|
||
//! Decoding never panics: a short buffer, an unknown byte order or a
|
||
//! dangling heap reference becomes [`Value::Error`].
|
||
|
||
use std::cell::RefCell;
|
||
|
||
use clawhdf5_format::datatype::{Datatype, DatatypeByteOrder, ReferenceType, StringPadding};
|
||
use clawhdf5_format::vl_data::{VlResolver, check_element_size};
|
||
use serde_json::Value as J;
|
||
|
||
use crate::dtype;
|
||
use crate::h5::H5;
|
||
|
||
#[derive(Debug, Clone, PartialEq)]
|
||
pub enum Value {
|
||
Int(i128),
|
||
/// A float and the width (in bits) it was stored with, so it prints at
|
||
/// its own precision.
|
||
Float(f64, u8),
|
||
Str(String),
|
||
/// A null variable-length string (heap address 0): h5dump prints it as
|
||
/// `NULL`, h5py reads it as empty.
|
||
NullStr,
|
||
/// Opaque, bitfield, time and oversized integers.
|
||
Bytes(Vec<u8>),
|
||
/// An enum member (name, when the value matches one) and its value.
|
||
Enum(Option<String>, i128),
|
||
Compound(Vec<(String, Value)>),
|
||
Array(Vec<Value>),
|
||
/// A variable-length sequence.
|
||
Seq(Vec<Value>),
|
||
/// A reference: the referenced object's address (`None` = null).
|
||
Ref(Option<u64>),
|
||
/// A region or attribute reference, kept as its bytes.
|
||
OtherRef(Vec<u8>),
|
||
Error(String),
|
||
}
|
||
|
||
pub fn hex(b: &[u8]) -> String {
|
||
let mut s = String::from("0x");
|
||
for x in b {
|
||
s.push_str(&format!("{x:02x}"));
|
||
}
|
||
s
|
||
}
|
||
|
||
/// Element bytes as an unsigned integer (at most 16 bytes).
|
||
fn bits(b: &[u8], order: &DatatypeByteOrder) -> Option<u128> {
|
||
if b.len() > 16 {
|
||
return None;
|
||
}
|
||
let mut v = 0u128;
|
||
match order {
|
||
DatatypeByteOrder::LittleEndian => {
|
||
for (i, x) in b.iter().enumerate() {
|
||
v |= u128::from(*x) << (8 * i);
|
||
}
|
||
}
|
||
DatatypeByteOrder::BigEndian => {
|
||
for x in b {
|
||
v = (v << 8) | u128::from(*x);
|
||
}
|
||
}
|
||
DatatypeByteOrder::Vax => return None,
|
||
}
|
||
Some(v)
|
||
}
|
||
|
||
/// The value of an integer (fixed-point) element, honouring its bit offset
|
||
/// and precision. `None` when it cannot be represented (over 16 bytes) or
|
||
/// `dt` is not an integer.
|
||
pub fn decode_int(dt: &Datatype, b: &[u8]) -> Option<i128> {
|
||
let Datatype::FixedPoint {
|
||
size,
|
||
byte_order,
|
||
signed,
|
||
bit_offset,
|
||
bit_precision,
|
||
} = dt
|
||
else {
|
||
return None;
|
||
};
|
||
let size = usize::try_from(*size).ok()?;
|
||
let v = bits(b.get(..size)?, byte_order)?;
|
||
let off = u32::from(*bit_offset);
|
||
let prec = u32::from(*bit_precision).min(128);
|
||
if off >= 128 || prec == 0 {
|
||
return Some(0);
|
||
}
|
||
let mut x = v >> off;
|
||
if prec < 128 {
|
||
x &= (1u128 << prec) - 1;
|
||
}
|
||
if *signed && prec < 128 && (x >> (prec - 1)) & 1 == 1 {
|
||
x |= !0u128 << prec;
|
||
return Some(x as i128);
|
||
}
|
||
if !*signed && prec == 128 && x > i128::MAX as u128 {
|
||
return None;
|
||
}
|
||
Some(x as i128)
|
||
}
|
||
|
||
fn decode_float(dt: &Datatype, b: &[u8]) -> Value {
|
||
let Datatype::FloatingPoint {
|
||
size, byte_order, ..
|
||
} = dt
|
||
else {
|
||
return Value::Error("not a float".into());
|
||
};
|
||
let Ok(n) = usize::try_from(*size) else {
|
||
return Value::Error("float size".into());
|
||
};
|
||
let Some(b) = b.get(..n) else {
|
||
return Value::Error("short element".into());
|
||
};
|
||
if dtype::is_ieee(dt)
|
||
&& let Some(v) = bits(b, byte_order)
|
||
{
|
||
return match n {
|
||
2 => Value::Float(
|
||
f64::from(clawhdf5_format::float16::f16_bits_to_f32(v as u16)),
|
||
16,
|
||
),
|
||
4 => Value::Float(f64::from(f32::from_bits(v as u32)), 32),
|
||
_ => Value::Float(f64::from_bits(v as u64), 64),
|
||
};
|
||
}
|
||
// Non-IEEE layouts (N-Bit floats, VAX order): the library converts.
|
||
match clawhdf5_format::data_read::read_as_f64(b, dt) {
|
||
Ok(v) if v.len() == 1 => Value::Float(v[0], (n * 8).min(64) as u8),
|
||
Ok(_) => Value::Error("float conversion".into()),
|
||
Err(e) => Value::Error(e.to_string()),
|
||
}
|
||
}
|
||
|
||
fn trim_string(b: &[u8], pad: &StringPadding) -> String {
|
||
let cut = b.iter().position(|&c| c == 0).unwrap_or(b.len());
|
||
let mut s = &b[..cut];
|
||
if matches!(pad, StringPadding::SpacePad) {
|
||
while let [rest @ .., b' '] = s {
|
||
s = rest;
|
||
}
|
||
}
|
||
String::from_utf8_lossy(s).into_owned()
|
||
}
|
||
|
||
/// Decodes elements of one file.
|
||
pub struct Decoder<'a> {
|
||
pub h5: &'a H5,
|
||
/// Variable-length elements are resolved as the library resolves them
|
||
/// (so as libhdf5 does), not by a decoder of our own.
|
||
vl: RefCell<VlResolver<'a>>,
|
||
}
|
||
|
||
impl<'a> Decoder<'a> {
|
||
pub fn new(h5: &'a H5) -> Self {
|
||
Self {
|
||
h5,
|
||
vl: RefCell::new(VlResolver::new(h5.data(), h5.os(), h5.ls())),
|
||
}
|
||
}
|
||
|
||
/// Decode element `i` of `raw`, an array of `dt` elements.
|
||
pub fn element(&self, dt: &Datatype, raw: &[u8], i: usize) -> Value {
|
||
let size = dt.type_size() as usize;
|
||
match i
|
||
.checked_mul(size)
|
||
.and_then(|s| raw.get(s..s.checked_add(size)?))
|
||
{
|
||
Some(b) => self.decode(dt, b, 0),
|
||
None => Value::Error("element out of range".into()),
|
||
}
|
||
}
|
||
|
||
pub fn decode(&self, dt: &Datatype, b: &[u8], depth: u32) -> Value {
|
||
if depth > 32 {
|
||
return Value::Error("datatype nesting too deep".into());
|
||
}
|
||
let size = dt.type_size() as usize;
|
||
let Some(b) = b.get(..size) else {
|
||
return Value::Error("short element".into());
|
||
};
|
||
match dt {
|
||
Datatype::FixedPoint { .. } => match decode_int(dt, b) {
|
||
Some(v) => Value::Int(v),
|
||
None => Value::Bytes(b.to_vec()),
|
||
},
|
||
Datatype::FloatingPoint { .. } => decode_float(dt, b),
|
||
Datatype::Time { .. } | Datatype::BitField { .. } | Datatype::Opaque { .. } => {
|
||
Value::Bytes(b.to_vec())
|
||
}
|
||
Datatype::String { padding, .. } => Value::Str(trim_string(b, padding)),
|
||
Datatype::Compound { members, .. } => {
|
||
let mut out = Vec::with_capacity(members.len());
|
||
for m in members {
|
||
let off = usize::try_from(m.byte_offset).unwrap_or(usize::MAX);
|
||
let v = match b.get(off..) {
|
||
Some(mb) => self.decode(&m.datatype, mb, depth + 1),
|
||
None => Value::Error("member out of bounds".into()),
|
||
};
|
||
out.push((m.name.clone(), v));
|
||
}
|
||
Value::Compound(out)
|
||
}
|
||
Datatype::Reference { ref_type, .. } => match ref_type {
|
||
ReferenceType::Object | ReferenceType::Object2 => {
|
||
match clawhdf5_format::data_read::read_object_references(b, dt, self.h5.os()) {
|
||
Ok(r) if r.len() == 1 => {
|
||
let a = r[0].address;
|
||
let undef = a == u64::MAX
|
||
|| (self.h5.os() < 8 && a == (1u64 << (8 * self.h5.os())) - 1);
|
||
Value::Ref(if undef || a == 0 { None } else { Some(a) })
|
||
}
|
||
Ok(_) => Value::Error("reference".into()),
|
||
Err(e) => Value::Error(e.to_string()),
|
||
}
|
||
}
|
||
_ => Value::OtherRef(b.to_vec()),
|
||
},
|
||
Datatype::Enumeration {
|
||
base_type, members, ..
|
||
} => {
|
||
let Some(v) = decode_int(base_type, b) else {
|
||
return Value::Bytes(b.to_vec());
|
||
};
|
||
let bs = base_type.type_size() as usize;
|
||
let name = members
|
||
.iter()
|
||
.find(|m| m.value.get(..bs) == b.get(..bs))
|
||
.map(|m| m.name.clone());
|
||
Value::Enum(name, v)
|
||
}
|
||
Datatype::Array {
|
||
base_type,
|
||
dimensions,
|
||
} => {
|
||
let n = dimensions
|
||
.iter()
|
||
.try_fold(1usize, |a, &d| a.checked_mul(d as usize));
|
||
let bs = base_type.type_size() as usize;
|
||
let Some(n) = n.filter(|n| n.checked_mul(bs).is_some_and(|t| t <= b.len())) else {
|
||
return Value::Error("array larger than its element".into());
|
||
};
|
||
let mut out = Vec::with_capacity(n);
|
||
for k in 0..n {
|
||
out.push(self.decode(base_type, &b[k * bs..], depth + 1));
|
||
}
|
||
Value::Array(out)
|
||
}
|
||
Datatype::VariableLength {
|
||
size,
|
||
is_string,
|
||
base_type,
|
||
..
|
||
} => match check_element_size(*size, self.h5.os()) {
|
||
Ok(()) => self.decode_vlen(*is_string, base_type, b, depth),
|
||
Err(e) => Value::Error(e.to_string()),
|
||
},
|
||
}
|
||
}
|
||
|
||
/// A variable-length element, resolved by the library's
|
||
/// [`VlResolver`]: a string ends at its first NUL, a heap object whose
|
||
/// size is not the element's length × base size is an error, and a
|
||
/// heap address of 0 is null — all as libhdf5 (and so h5dump and h5py)
|
||
/// has it.
|
||
fn decode_vlen(&self, is_string: bool, base: &Datatype, b: &[u8], depth: u32) -> Value {
|
||
if is_string {
|
||
return match self.vl.borrow_mut().string_element(b) {
|
||
Ok(Some(s)) => Value::Str(String::from_utf8_lossy(s).into_owned()),
|
||
Ok(None) => Value::NullStr,
|
||
Err(e) => Value::Error(e.to_string()),
|
||
};
|
||
}
|
||
let bs = base.type_size() as usize;
|
||
if bs == 0 {
|
||
return Value::Error("VL base type of size 0".into());
|
||
}
|
||
let obj = match self.vl.borrow_mut().element(b, bs) {
|
||
Ok(o) => o.unwrap_or(&[]),
|
||
Err(e) => return Value::Error(e.to_string()),
|
||
};
|
||
Value::Seq(
|
||
obj.chunks_exact(bs)
|
||
.map(|e| self.decode(base, e, depth + 1))
|
||
.collect(),
|
||
)
|
||
}
|
||
}
|
||
|
||
/// Format a float like C's `%g` would at full round-trip precision: plain
|
||
/// digits for moderate magnitudes, an exponent otherwise.
|
||
pub fn fmt_float(v: f64, width: u8) -> String {
|
||
if v.is_nan() {
|
||
return "NaN".into();
|
||
}
|
||
if v.is_infinite() {
|
||
return if v > 0.0 { "Inf".into() } else { "-Inf".into() };
|
||
}
|
||
let a = v.abs();
|
||
let plain = a == 0.0 || (1e-5..1e16).contains(&a);
|
||
match (width, plain) {
|
||
(16 | 32, true) => format!("{}", v as f32),
|
||
(16 | 32, false) => format!("{:e}", v as f32),
|
||
(_, true) => format!("{v}"),
|
||
(_, false) => format!("{v:e}"),
|
||
}
|
||
}
|
||
|
||
fn escape(s: &str) -> String {
|
||
let mut o = String::with_capacity(s.len() + 2);
|
||
for c in s.chars() {
|
||
match c {
|
||
'"' => o.push_str("\\\""),
|
||
'\\' => o.push_str("\\\\"),
|
||
'\n' => o.push_str("\\n"),
|
||
'\r' => o.push_str("\\r"),
|
||
'\t' => o.push_str("\\t"),
|
||
c if (c as u32) < 0x20 => o.push_str(&format!("\\{:03o}", c as u32)),
|
||
c => o.push(c),
|
||
}
|
||
}
|
||
o
|
||
}
|
||
|
||
/// A fixed-length string element's bytes quoted as h5dump prints a
|
||
/// null-padded string: every byte, NULs as `\000`.
|
||
pub fn quote_bytes(b: &[u8]) -> String {
|
||
format!("\"{}\"", escape(&String::from_utf8_lossy(b)))
|
||
}
|
||
|
||
/// Text form, as in an h5dump DATA block.
|
||
pub fn text(v: &Value, h5paths: &dyn Fn(u64) -> Option<String>) -> String {
|
||
match v {
|
||
Value::Int(i) => i.to_string(),
|
||
Value::Float(f, w) => fmt_float(*f, *w),
|
||
Value::Str(s) => format!("\"{}\"", escape(s)),
|
||
Value::NullStr => "NULL".into(),
|
||
Value::Bytes(b) => hex(b),
|
||
Value::Enum(Some(n), _) => n.clone(),
|
||
Value::Enum(None, i) => i.to_string(),
|
||
Value::Compound(ms) => format!(
|
||
"{{ {} }}",
|
||
ms.iter()
|
||
.map(|(_, v)| text(v, h5paths))
|
||
.collect::<Vec<_>>()
|
||
.join(", ")
|
||
),
|
||
Value::Array(vs) => format!(
|
||
"[ {} ]",
|
||
vs.iter()
|
||
.map(|v| text(v, h5paths))
|
||
.collect::<Vec<_>>()
|
||
.join(", ")
|
||
),
|
||
Value::Seq(vs) => format!(
|
||
"({})",
|
||
vs.iter()
|
||
.map(|v| text(v, h5paths))
|
||
.collect::<Vec<_>>()
|
||
.join(", ")
|
||
),
|
||
Value::Ref(None) => "NULL".into(),
|
||
Value::Ref(Some(a)) => match h5paths(*a) {
|
||
Some(p) => format!("\"{p}\""),
|
||
None => format!("{a:#x}"),
|
||
},
|
||
Value::OtherRef(b) => hex(b),
|
||
Value::Error(e) => format!("<error: {e}>"),
|
||
}
|
||
}
|
||
|
||
/// hdf5-json value form.
|
||
pub fn to_json(v: &Value, h5paths: &dyn Fn(u64) -> Option<String>) -> J {
|
||
match v {
|
||
Value::Int(i) => {
|
||
if let Ok(x) = i64::try_from(*i) {
|
||
J::from(x)
|
||
} else if let Ok(x) = u64::try_from(*i) {
|
||
J::from(x)
|
||
} else {
|
||
J::from(i.to_string())
|
||
}
|
||
}
|
||
Value::Float(f, _) => {
|
||
if f.is_finite() {
|
||
serde_json::Number::from_f64(*f)
|
||
.map(J::Number)
|
||
.unwrap_or(J::Null)
|
||
} else if f.is_nan() {
|
||
J::from("NaN")
|
||
} else if *f > 0.0 {
|
||
J::from("Infinity")
|
||
} else {
|
||
J::from("-Infinity")
|
||
}
|
||
}
|
||
Value::Str(s) => J::from(s.as_str()),
|
||
Value::NullStr => J::from(""),
|
||
Value::Bytes(b) | Value::OtherRef(b) => J::from(hex(b)),
|
||
Value::Enum(_, i) => to_json(&Value::Int(*i), h5paths),
|
||
Value::Compound(ms) => J::Array(ms.iter().map(|(_, v)| to_json(v, h5paths)).collect()),
|
||
Value::Array(vs) | Value::Seq(vs) => {
|
||
J::Array(vs.iter().map(|v| to_json(v, h5paths)).collect())
|
||
}
|
||
Value::Ref(None) => J::Null,
|
||
Value::Ref(Some(a)) => J::from(h5paths(*a).unwrap_or_else(|| format!("{a:#x}"))),
|
||
Value::Error(e) => serde_json::json!({ "error": e }),
|
||
}
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
|
||
fn int(size: u32, signed: bool, order: DatatypeByteOrder, off: u16, prec: u16) -> Datatype {
|
||
Datatype::FixedPoint {
|
||
size,
|
||
byte_order: order,
|
||
signed,
|
||
bit_offset: off,
|
||
bit_precision: prec,
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn integers_decode_with_order_offset_and_sign() {
|
||
let le = DatatypeByteOrder::LittleEndian;
|
||
let be = DatatypeByteOrder::BigEndian;
|
||
assert_eq!(
|
||
decode_int(&int(2, true, le.clone(), 0, 16), &[0xff, 0xff]),
|
||
Some(-1)
|
||
);
|
||
assert_eq!(
|
||
decode_int(&int(2, false, le, 0, 16), &[0xff, 0xff]),
|
||
Some(65535)
|
||
);
|
||
assert_eq!(
|
||
decode_int(&int(2, true, be.clone(), 0, 16), &[0x80, 0x00]),
|
||
Some(-32768)
|
||
);
|
||
// 17-bit signed field at offset 4: -5
|
||
let stored = (((-5i32) as u32) & 0x1_FFFF) << 4;
|
||
assert_eq!(
|
||
decode_int(&int(4, true, be, 4, 17), &stored.to_be_bytes()),
|
||
Some(-5)
|
||
);
|
||
assert_eq!(
|
||
decode_int(&int(4, true, DatatypeByteOrder::Vax, 0, 32), &[0; 4]),
|
||
None
|
||
);
|
||
assert_eq!(
|
||
decode_int(
|
||
&int(4, true, DatatypeByteOrder::LittleEndian, 0, 32),
|
||
&[0; 2]
|
||
),
|
||
None
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn floats_print_at_their_own_precision() {
|
||
assert_eq!(fmt_float(f64::from(0.1f32), 32), "0.1");
|
||
assert_eq!(fmt_float(0.1, 64), "0.1");
|
||
assert_eq!(fmt_float(1e20, 64), "1e20");
|
||
assert_eq!(fmt_float(2.0, 64), "2");
|
||
assert_eq!(fmt_float(f64::NAN, 64), "NaN");
|
||
}
|
||
}
|