quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases 1 Wiki Activity

fix(tools,wasm): resolve VL data through the library's VlResolver

Browse Source 
h5rs (dump, ls, diff, check --data) kept its own lenient VL decoder:
a heap object longer than its element was cut to the element's length
(libhdf5 and h5py refuse it), a null string printed "" where h5dump
prints NULL, the stored element size was trusted, and every heap
collection was kept as an owned copy for the whole run. It now resolves
each element with VlResolver::element / string_element (new: one element
in place, borrowing from the file), and refuses a VL type whose stored
element size is not 4 + offset size + 4, as File does. H5::heap_object
and its cache are gone. h5diff compares a null VL string equal to an
empty one; so does h5rs diff.

clawhdf5-wasm already resolved VL strings with read_vl_strings; it now
uses VlResolver and checks the stored element size before reading, as
File::read_string does.

Tests (h5py writes the files, patched for "a\0b", a null element and
mis-sized heap objects, with 8- and 4-byte offsets):
- h5rs_interop dump_prints_vl_data_like_h5dump: byte-identical to h5dump;
- dump_json_vl_values_match_h5py: h5py's values, errors where h5py fails;
- check_data_flags_mis_sized_vl_heap_objects;
- clawhdf5-wasm tests/vl_strings.rs: wasm, File and h5py agree.
All four fail before. check --data over the 150 cve_hdf5 CVE and fuzzer
files now passes 15 (h5dump rejects 8 of them), was 16 and 9: the
stored-size check flags cve-2024-32608. h5rs-check-ok-files.sh --data:
0 of 422 flagged; h5rs-fuzz.sh: clean on 180 files.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-26 09:04:21 -05:00
co-authored by Claude Opus 5.5
parent 41b7837d0a
commit d345ffbf80
11 changed files with 533 additions and 152 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-tools/src/value.rs
+43 -55
View File
@@ -3,7 +3,10 @@
//! Decoding never panics: a short buffer, an unknown byte order or a
//! dangling heap reference becomes [`Value::Error`].
use std::cell::RefCell;
use clawhdf5_format::datatype::{Datatype, DatatypeByteOrder, ReferenceType, StringPadding};
use clawhdf5_format::vl_data::{VlResolver, check_element_size};
use serde_json::Value as J;
use crate::dtype;
@@ -16,6 +19,9 @@ pub enum Value {
/// its own precision.
Float(f64, u8),
Str(String),
/// A null variable-length string (heap address 0): h5dump prints it as
/// `NULL`, h5py reads it as empty.
NullStr,
/// Opaque, bitfield, time and oversized integers.
Bytes(Vec<u8>),
/// An enum member (name, when the value matches one) and its value.
@@ -129,10 +135,10 @@ fn decode_float(dt: &Datatype, b: &[u8]) -> Value {
}
}
fn trim_string(b: &[u8], pad: Option<&StringPadding>) -> String {
fn trim_string(b: &[u8], pad: &StringPadding) -> String {
let cut = b.iter().position(|&c| c == 0).unwrap_or(b.len());
let mut s = &b[..cut];
if matches!(pad, Some(StringPadding::SpacePad)) {
if matches!(pad, StringPadding::SpacePad) {
while let [rest @ .., b' '] = s {
s = rest;
}
@@ -140,22 +146,20 @@ fn trim_string(b: &[u8], pad: Option<&StringPadding>) -> String {
String::from_utf8_lossy(s).into_owned()
}
/// Little-endian unsigned integer of `b` (up to 8 bytes).
fn le(b: &[u8]) -> u64 {
b.iter()
.take(8)
.enumerate()
.fold(0u64, |a, (i, &x)| a | (u64::from(x) << (8 * i)))
}
/// Decodes elements of one file.
pub struct Decoder<'a> {
pub h5: &'a H5,
/// Variable-length elements are resolved as the library resolves them
/// (so as libhdf5 does), not by a decoder of our own.
vl: RefCell<VlResolver<'a>>,
}
impl<'a> Decoder<'a> {
pub fn new(h5: &'a H5) -> Self {
Self { h5 }
Self {
h5,
vl: RefCell::new(VlResolver::new(h5.data(), h5.os(), h5.ls())),
}
}
/// Decode element `i` of `raw`, an array of `dt` elements.
@@ -187,7 +191,7 @@ impl<'a> Decoder<'a> {
Datatype::Time { .. } | Datatype::BitField { .. } | Datatype::Opaque { .. } => {
Value::Bytes(b.to_vec())
}
Datatype::String { padding, .. } => Value::Str(trim_string(b, Some(padding))),
Datatype::String { padding, .. } => Value::Str(trim_string(b, padding)),
Datatype::Compound { members, .. } => {
let mut out = Vec::with_capacity(members.len());
for m in members {
@@ -246,61 +250,43 @@ impl<'a> Decoder<'a> {
Value::Array(out)
}
Datatype::VariableLength {
size,
is_string,
padding,
base_type,
..
} => self.decode_vlen(*is_string, padding.as_ref(), base_type, b, depth),
} => match check_element_size(*size, self.h5.os()) {
Ok(()) => self.decode_vlen(*is_string, base_type, b, depth),
Err(e) => Value::Error(e.to_string()),
},
}
}
fn decode_vlen(
&self,
is_string: bool,
padding: Option<&StringPadding>,
base: &Datatype,
b: &[u8],
depth: u32,
) -> Value {
let os = usize::from(self.h5.os());
let (Some(lenb), Some(addrb), Some(idxb)) =
(b.get(..4), b.get(4..4 + os), b.get(4 + os..8 + os))
else {
return Value::Error("short VL element".into());
};
let len = le(lenb) as usize;
let addr = le(addrb);
let idx = le(idxb) as u32;
let undef = if os >= 8 {
u64::MAX
} else {
(1u64 << (8 * os)) - 1
};
let obj = if len == 0 || addr == 0 || addr == undef {
Vec::new()
} else {
match self.h5.heap_object(addr, idx) {
Ok(o) => o,
Err(e) => return Value::Error(e.to_string()),
}
};
/// A variable-length element, resolved by the library's
/// [`VlResolver`]: a string ends at its first NUL, a heap object whose
/// size is not the element's length × base size is an error, and a
/// heap address of 0 is null — all as libhdf5 (and so h5dump and h5py)
/// has it.
fn decode_vlen(&self, is_string: bool, base: &Datatype, b: &[u8], depth: u32) -> Value {
if is_string {
let l = len.min(obj.len());
return Value::Str(trim_string(&obj[..l], padding));
return match self.vl.borrow_mut().string_element(b) {
Ok(Some(s)) => Value::Str(String::from_utf8_lossy(s).into_owned()),
Ok(None) => Value::NullStr,
Err(e) => Value::Error(e.to_string()),
};
}
let bs = base.type_size() as usize;
if bs == 0 {
return Value::Error("VL base type of size 0".into());
}
match len.checked_mul(bs) {
Some(need) if need <= obj.len() => {}
_ => return Value::Error("VL sequence longer than its heap object".into()),
}
let mut out = Vec::with_capacity(len);
for k in 0..len {
out.push(self.decode(base, &obj[k * bs..], depth + 1));
}
Value::Seq(out)
let obj = match self.vl.borrow_mut().element(b, bs) {
Ok(o) => o.unwrap_or(&[]),
Err(e) => return Value::Error(e.to_string()),
};
Value::Seq(
obj.chunks_exact(bs)
.map(|e| self.decode(base, e, depth + 1))
.collect(),
)
}
}
@@ -351,6 +337,7 @@ pub fn text(v: &Value, h5paths: &dyn Fn(u64) -> Option<String>) -> String {
Value::Int(i) => i.to_string(),
Value::Float(f, w) => fmt_float(*f, *w),
Value::Str(s) => format!("\"{}\"", escape(s)),
Value::NullStr => "NULL".into(),
Value::Bytes(b) => hex(b),
Value::Enum(Some(n), _) => n.clone(),
Value::Enum(None, i) => i.to_string(),
@@ -411,6 +398,7 @@ pub fn to_json(v: &Value, h5paths: &dyn Fn(u64) -> Option<String>) -> J {
}
}
Value::Str(s) => J::from(s.as_str()),
Value::NullStr => J::from(""),
Value::Bytes(b) | Value::OtherRef(b) => J::from(hex(b)),
Value::Enum(_, i) => to_json(&Value::Int(*i), h5paths),
Value::Compound(ms) => J::Array(ms.iter().map(|(_, v)| to_json(v, h5paths)).collect()),