feat(tools): h5rs, pure-Rust HDF5 tools (ls, dump, stat, diff, check)
New workspace crate clawhdf5-tools with one binary, h5rs, built only on the clawhdf5 facade and clawhdf5-format (no libhdf5, no C): - ls [-r] [-v] FILE[/path]: h5ls's listing (same text in its first two columns) plus the datatype; -v adds address, link count, layout and chunk index, chunk size, storage, filters, datatype and attributes. - dump [--json] [-A] [-p] [-d PATH] FILE: h5dump DDL (byte-identical to h5dump 1.14.6 on the test files) or hdf5-json. - stat FILE: h5stat's object/link/rank/layout/filter/attribute counts, raw data and total size. - diff [-r] [-q] [-d D] [-p R] A B [OBJ1 [OBJ2]]: structural and value differences, exit 0/1/2 like h5diff. - check [--data] FILE: walks every object, parses every message, verifies the checksums of every v2+ structure (including the fractal heap blocks the library never checks), checks chunk indexes against their datasets and raw data for out-of-file or overlapping extents; every problem with its address. Values over --max-bytes are reported, not read; dense-storage heaps are verified before objects are read from them; panics are caught (exit 3). Tests compare with h5ls, h5stat, h5dump and h5diff and with h5py's values, and flip the checksum of every checksummed structure in a v1.14-format file. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
@@ -0,0 +1,483 @@
|
||||
//! Decoding one element of any datatype into a [`Value`], and printing it.
|
||||
//!
|
||||
//! Decoding never panics: a short buffer, an unknown byte order or a
|
||||
//! dangling heap reference becomes [`Value::Error`].
|
||||
|
||||
use clawhdf5_format::datatype::{Datatype, DatatypeByteOrder, ReferenceType, StringPadding};
|
||||
use serde_json::Value as J;
|
||||
|
||||
use crate::dtype;
|
||||
use crate::h5::H5;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum Value {
|
||||
Int(i128),
|
||||
/// A float and the width (in bits) it was stored with, so it prints at
|
||||
/// its own precision.
|
||||
Float(f64, u8),
|
||||
Str(String),
|
||||
/// Opaque, bitfield, time and oversized integers.
|
||||
Bytes(Vec<u8>),
|
||||
/// An enum member (name, when the value matches one) and its value.
|
||||
Enum(Option<String>, i128),
|
||||
Compound(Vec<(String, Value)>),
|
||||
Array(Vec<Value>),
|
||||
/// A variable-length sequence.
|
||||
Seq(Vec<Value>),
|
||||
/// A reference: the referenced object's address (`None` = null).
|
||||
Ref(Option<u64>),
|
||||
/// A region or attribute reference, kept as its bytes.
|
||||
OtherRef(Vec<u8>),
|
||||
Error(String),
|
||||
}
|
||||
|
||||
pub fn hex(b: &[u8]) -> String {
|
||||
let mut s = String::from("0x");
|
||||
for x in b {
|
||||
s.push_str(&format!("{x:02x}"));
|
||||
}
|
||||
s
|
||||
}
|
||||
|
||||
/// Element bytes as an unsigned integer (at most 16 bytes).
|
||||
fn bits(b: &[u8], order: &DatatypeByteOrder) -> Option<u128> {
|
||||
if b.len() > 16 {
|
||||
return None;
|
||||
}
|
||||
let mut v = 0u128;
|
||||
match order {
|
||||
DatatypeByteOrder::LittleEndian => {
|
||||
for (i, x) in b.iter().enumerate() {
|
||||
v |= u128::from(*x) << (8 * i);
|
||||
}
|
||||
}
|
||||
DatatypeByteOrder::BigEndian => {
|
||||
for x in b {
|
||||
v = (v << 8) | u128::from(*x);
|
||||
}
|
||||
}
|
||||
DatatypeByteOrder::Vax => return None,
|
||||
}
|
||||
Some(v)
|
||||
}
|
||||
|
||||
/// The value of an integer (fixed-point) element, honouring its bit offset
|
||||
/// and precision. `None` when it cannot be represented (over 16 bytes) or
|
||||
/// `dt` is not an integer.
|
||||
pub fn decode_int(dt: &Datatype, b: &[u8]) -> Option<i128> {
|
||||
let Datatype::FixedPoint {
|
||||
size,
|
||||
byte_order,
|
||||
signed,
|
||||
bit_offset,
|
||||
bit_precision,
|
||||
} = dt
|
||||
else {
|
||||
return None;
|
||||
};
|
||||
let size = usize::try_from(*size).ok()?;
|
||||
let v = bits(b.get(..size)?, byte_order)?;
|
||||
let off = u32::from(*bit_offset);
|
||||
let prec = u32::from(*bit_precision).min(128);
|
||||
if off >= 128 || prec == 0 {
|
||||
return Some(0);
|
||||
}
|
||||
let mut x = v >> off;
|
||||
if prec < 128 {
|
||||
x &= (1u128 << prec) - 1;
|
||||
}
|
||||
if *signed && prec < 128 && (x >> (prec - 1)) & 1 == 1 {
|
||||
x |= !0u128 << prec;
|
||||
return Some(x as i128);
|
||||
}
|
||||
if !*signed && prec == 128 && x > i128::MAX as u128 {
|
||||
return None;
|
||||
}
|
||||
Some(x as i128)
|
||||
}
|
||||
|
||||
fn decode_float(dt: &Datatype, b: &[u8]) -> Value {
|
||||
let Datatype::FloatingPoint {
|
||||
size, byte_order, ..
|
||||
} = dt
|
||||
else {
|
||||
return Value::Error("not a float".into());
|
||||
};
|
||||
let Ok(n) = usize::try_from(*size) else {
|
||||
return Value::Error("float size".into());
|
||||
};
|
||||
let Some(b) = b.get(..n) else {
|
||||
return Value::Error("short element".into());
|
||||
};
|
||||
if dtype::is_ieee(dt)
|
||||
&& let Some(v) = bits(b, byte_order)
|
||||
{
|
||||
return match n {
|
||||
2 => Value::Float(
|
||||
f64::from(clawhdf5_format::float16::f16_bits_to_f32(v as u16)),
|
||||
16,
|
||||
),
|
||||
4 => Value::Float(f64::from(f32::from_bits(v as u32)), 32),
|
||||
_ => Value::Float(f64::from_bits(v as u64), 64),
|
||||
};
|
||||
}
|
||||
// Non-IEEE layouts (N-Bit floats, VAX order): the library converts.
|
||||
match clawhdf5_format::data_read::read_as_f64(b, dt) {
|
||||
Ok(v) if v.len() == 1 => Value::Float(v[0], (n * 8).min(64) as u8),
|
||||
Ok(_) => Value::Error("float conversion".into()),
|
||||
Err(e) => Value::Error(e.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
fn trim_string(b: &[u8], pad: Option<&StringPadding>) -> String {
|
||||
let cut = b.iter().position(|&c| c == 0).unwrap_or(b.len());
|
||||
let mut s = &b[..cut];
|
||||
if matches!(pad, Some(StringPadding::SpacePad)) {
|
||||
while let [rest @ .., b' '] = s {
|
||||
s = rest;
|
||||
}
|
||||
}
|
||||
String::from_utf8_lossy(s).into_owned()
|
||||
}
|
||||
|
||||
/// Little-endian unsigned integer of `b` (up to 8 bytes).
|
||||
fn le(b: &[u8]) -> u64 {
|
||||
b.iter()
|
||||
.take(8)
|
||||
.enumerate()
|
||||
.fold(0u64, |a, (i, &x)| a | (u64::from(x) << (8 * i)))
|
||||
}
|
||||
|
||||
/// Decodes elements of one file.
|
||||
pub struct Decoder<'a> {
|
||||
pub h5: &'a H5,
|
||||
}
|
||||
|
||||
impl<'a> Decoder<'a> {
|
||||
pub fn new(h5: &'a H5) -> Self {
|
||||
Self { h5 }
|
||||
}
|
||||
|
||||
/// Decode element `i` of `raw`, an array of `dt` elements.
|
||||
pub fn element(&self, dt: &Datatype, raw: &[u8], i: usize) -> Value {
|
||||
let size = dt.type_size() as usize;
|
||||
match i
|
||||
.checked_mul(size)
|
||||
.and_then(|s| raw.get(s..s.checked_add(size)?))
|
||||
{
|
||||
Some(b) => self.decode(dt, b, 0),
|
||||
None => Value::Error("element out of range".into()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn decode(&self, dt: &Datatype, b: &[u8], depth: u32) -> Value {
|
||||
if depth > 32 {
|
||||
return Value::Error("datatype nesting too deep".into());
|
||||
}
|
||||
let size = dt.type_size() as usize;
|
||||
let Some(b) = b.get(..size) else {
|
||||
return Value::Error("short element".into());
|
||||
};
|
||||
match dt {
|
||||
Datatype::FixedPoint { .. } => match decode_int(dt, b) {
|
||||
Some(v) => Value::Int(v),
|
||||
None => Value::Bytes(b.to_vec()),
|
||||
},
|
||||
Datatype::FloatingPoint { .. } => decode_float(dt, b),
|
||||
Datatype::Time { .. } | Datatype::BitField { .. } | Datatype::Opaque { .. } => {
|
||||
Value::Bytes(b.to_vec())
|
||||
}
|
||||
Datatype::String { padding, .. } => Value::Str(trim_string(b, Some(padding))),
|
||||
Datatype::Compound { members, .. } => {
|
||||
let mut out = Vec::with_capacity(members.len());
|
||||
for m in members {
|
||||
let off = usize::try_from(m.byte_offset).unwrap_or(usize::MAX);
|
||||
let v = match b.get(off..) {
|
||||
Some(mb) => self.decode(&m.datatype, mb, depth + 1),
|
||||
None => Value::Error("member out of bounds".into()),
|
||||
};
|
||||
out.push((m.name.clone(), v));
|
||||
}
|
||||
Value::Compound(out)
|
||||
}
|
||||
Datatype::Reference { ref_type, .. } => match ref_type {
|
||||
ReferenceType::Object | ReferenceType::Object2 => {
|
||||
match clawhdf5_format::data_read::read_object_references(b, dt, self.h5.os()) {
|
||||
Ok(r) if r.len() == 1 => {
|
||||
let a = r[0].address;
|
||||
let undef = a == u64::MAX
|
||||
|| (self.h5.os() < 8 && a == (1u64 << (8 * self.h5.os())) - 1);
|
||||
Value::Ref(if undef || a == 0 { None } else { Some(a) })
|
||||
}
|
||||
Ok(_) => Value::Error("reference".into()),
|
||||
Err(e) => Value::Error(e.to_string()),
|
||||
}
|
||||
}
|
||||
_ => Value::OtherRef(b.to_vec()),
|
||||
},
|
||||
Datatype::Enumeration {
|
||||
base_type, members, ..
|
||||
} => {
|
||||
let Some(v) = decode_int(base_type, b) else {
|
||||
return Value::Bytes(b.to_vec());
|
||||
};
|
||||
let bs = base_type.type_size() as usize;
|
||||
let name = members
|
||||
.iter()
|
||||
.find(|m| m.value.get(..bs) == b.get(..bs))
|
||||
.map(|m| m.name.clone());
|
||||
Value::Enum(name, v)
|
||||
}
|
||||
Datatype::Array {
|
||||
base_type,
|
||||
dimensions,
|
||||
} => {
|
||||
let n = dimensions
|
||||
.iter()
|
||||
.try_fold(1usize, |a, &d| a.checked_mul(d as usize));
|
||||
let bs = base_type.type_size() as usize;
|
||||
let Some(n) = n.filter(|n| n.checked_mul(bs).is_some_and(|t| t <= b.len())) else {
|
||||
return Value::Error("array larger than its element".into());
|
||||
};
|
||||
let mut out = Vec::with_capacity(n);
|
||||
for k in 0..n {
|
||||
out.push(self.decode(base_type, &b[k * bs..], depth + 1));
|
||||
}
|
||||
Value::Array(out)
|
||||
}
|
||||
Datatype::VariableLength {
|
||||
is_string,
|
||||
padding,
|
||||
base_type,
|
||||
..
|
||||
} => self.decode_vlen(*is_string, padding.as_ref(), base_type, b, depth),
|
||||
}
|
||||
}
|
||||
|
||||
fn decode_vlen(
|
||||
&self,
|
||||
is_string: bool,
|
||||
padding: Option<&StringPadding>,
|
||||
base: &Datatype,
|
||||
b: &[u8],
|
||||
depth: u32,
|
||||
) -> Value {
|
||||
let os = usize::from(self.h5.os());
|
||||
let (Some(lenb), Some(addrb), Some(idxb)) =
|
||||
(b.get(..4), b.get(4..4 + os), b.get(4 + os..8 + os))
|
||||
else {
|
||||
return Value::Error("short VL element".into());
|
||||
};
|
||||
let len = le(lenb) as usize;
|
||||
let addr = le(addrb);
|
||||
let idx = le(idxb) as u32;
|
||||
let undef = if os >= 8 {
|
||||
u64::MAX
|
||||
} else {
|
||||
(1u64 << (8 * os)) - 1
|
||||
};
|
||||
let obj = if len == 0 || addr == 0 || addr == undef {
|
||||
Vec::new()
|
||||
} else {
|
||||
match self.h5.heap_object(addr, idx) {
|
||||
Ok(o) => o,
|
||||
Err(e) => return Value::Error(e.to_string()),
|
||||
}
|
||||
};
|
||||
if is_string {
|
||||
let l = len.min(obj.len());
|
||||
return Value::Str(trim_string(&obj[..l], padding));
|
||||
}
|
||||
let bs = base.type_size() as usize;
|
||||
if bs == 0 {
|
||||
return Value::Error("VL base type of size 0".into());
|
||||
}
|
||||
match len.checked_mul(bs) {
|
||||
Some(need) if need <= obj.len() => {}
|
||||
_ => return Value::Error("VL sequence longer than its heap object".into()),
|
||||
}
|
||||
let mut out = Vec::with_capacity(len);
|
||||
for k in 0..len {
|
||||
out.push(self.decode(base, &obj[k * bs..], depth + 1));
|
||||
}
|
||||
Value::Seq(out)
|
||||
}
|
||||
}
|
||||
|
||||
/// Format a float like C's `%g` would at full round-trip precision: plain
|
||||
/// digits for moderate magnitudes, an exponent otherwise.
|
||||
pub fn fmt_float(v: f64, width: u8) -> String {
|
||||
if v.is_nan() {
|
||||
return "NaN".into();
|
||||
}
|
||||
if v.is_infinite() {
|
||||
return if v > 0.0 { "Inf".into() } else { "-Inf".into() };
|
||||
}
|
||||
let a = v.abs();
|
||||
let plain = a == 0.0 || (1e-5..1e16).contains(&a);
|
||||
match (width, plain) {
|
||||
(16 | 32, true) => format!("{}", v as f32),
|
||||
(16 | 32, false) => format!("{:e}", v as f32),
|
||||
(_, true) => format!("{v}"),
|
||||
(_, false) => format!("{v:e}"),
|
||||
}
|
||||
}
|
||||
|
||||
fn escape(s: &str) -> String {
|
||||
let mut o = String::with_capacity(s.len() + 2);
|
||||
for c in s.chars() {
|
||||
match c {
|
||||
'"' => o.push_str("\\\""),
|
||||
'\\' => o.push_str("\\\\"),
|
||||
'\n' => o.push_str("\\n"),
|
||||
'\r' => o.push_str("\\r"),
|
||||
'\t' => o.push_str("\\t"),
|
||||
c if (c as u32) < 0x20 => o.push_str(&format!("\\{:03o}", c as u32)),
|
||||
c => o.push(c),
|
||||
}
|
||||
}
|
||||
o
|
||||
}
|
||||
|
||||
/// A fixed-length string element's bytes quoted as h5dump prints a
|
||||
/// null-padded string: every byte, NULs as `\000`.
|
||||
pub fn quote_bytes(b: &[u8]) -> String {
|
||||
format!("\"{}\"", escape(&String::from_utf8_lossy(b)))
|
||||
}
|
||||
|
||||
/// Text form, as in an h5dump DATA block.
|
||||
pub fn text(v: &Value, h5paths: &dyn Fn(u64) -> Option<String>) -> String {
|
||||
match v {
|
||||
Value::Int(i) => i.to_string(),
|
||||
Value::Float(f, w) => fmt_float(*f, *w),
|
||||
Value::Str(s) => format!("\"{}\"", escape(s)),
|
||||
Value::Bytes(b) => hex(b),
|
||||
Value::Enum(Some(n), _) => n.clone(),
|
||||
Value::Enum(None, i) => i.to_string(),
|
||||
Value::Compound(ms) => format!(
|
||||
"{{ {} }}",
|
||||
ms.iter()
|
||||
.map(|(_, v)| text(v, h5paths))
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ")
|
||||
),
|
||||
Value::Array(vs) => format!(
|
||||
"[ {} ]",
|
||||
vs.iter()
|
||||
.map(|v| text(v, h5paths))
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ")
|
||||
),
|
||||
Value::Seq(vs) => format!(
|
||||
"({})",
|
||||
vs.iter()
|
||||
.map(|v| text(v, h5paths))
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ")
|
||||
),
|
||||
Value::Ref(None) => "NULL".into(),
|
||||
Value::Ref(Some(a)) => match h5paths(*a) {
|
||||
Some(p) => format!("\"{p}\""),
|
||||
None => format!("{a:#x}"),
|
||||
},
|
||||
Value::OtherRef(b) => hex(b),
|
||||
Value::Error(e) => format!("<error: {e}>"),
|
||||
}
|
||||
}
|
||||
|
||||
/// hdf5-json value form.
|
||||
pub fn to_json(v: &Value, h5paths: &dyn Fn(u64) -> Option<String>) -> J {
|
||||
match v {
|
||||
Value::Int(i) => {
|
||||
if let Ok(x) = i64::try_from(*i) {
|
||||
J::from(x)
|
||||
} else if let Ok(x) = u64::try_from(*i) {
|
||||
J::from(x)
|
||||
} else {
|
||||
J::from(i.to_string())
|
||||
}
|
||||
}
|
||||
Value::Float(f, _) => {
|
||||
if f.is_finite() {
|
||||
serde_json::Number::from_f64(*f)
|
||||
.map(J::Number)
|
||||
.unwrap_or(J::Null)
|
||||
} else if f.is_nan() {
|
||||
J::from("NaN")
|
||||
} else if *f > 0.0 {
|
||||
J::from("Infinity")
|
||||
} else {
|
||||
J::from("-Infinity")
|
||||
}
|
||||
}
|
||||
Value::Str(s) => J::from(s.as_str()),
|
||||
Value::Bytes(b) | Value::OtherRef(b) => J::from(hex(b)),
|
||||
Value::Enum(_, i) => to_json(&Value::Int(*i), h5paths),
|
||||
Value::Compound(ms) => J::Array(ms.iter().map(|(_, v)| to_json(v, h5paths)).collect()),
|
||||
Value::Array(vs) | Value::Seq(vs) => {
|
||||
J::Array(vs.iter().map(|v| to_json(v, h5paths)).collect())
|
||||
}
|
||||
Value::Ref(None) => J::Null,
|
||||
Value::Ref(Some(a)) => J::from(h5paths(*a).unwrap_or_else(|| format!("{a:#x}"))),
|
||||
Value::Error(e) => serde_json::json!({ "error": e }),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn int(size: u32, signed: bool, order: DatatypeByteOrder, off: u16, prec: u16) -> Datatype {
|
||||
Datatype::FixedPoint {
|
||||
size,
|
||||
byte_order: order,
|
||||
signed,
|
||||
bit_offset: off,
|
||||
bit_precision: prec,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn integers_decode_with_order_offset_and_sign() {
|
||||
let le = DatatypeByteOrder::LittleEndian;
|
||||
let be = DatatypeByteOrder::BigEndian;
|
||||
assert_eq!(
|
||||
decode_int(&int(2, true, le.clone(), 0, 16), &[0xff, 0xff]),
|
||||
Some(-1)
|
||||
);
|
||||
assert_eq!(
|
||||
decode_int(&int(2, false, le, 0, 16), &[0xff, 0xff]),
|
||||
Some(65535)
|
||||
);
|
||||
assert_eq!(
|
||||
decode_int(&int(2, true, be.clone(), 0, 16), &[0x80, 0x00]),
|
||||
Some(-32768)
|
||||
);
|
||||
// 17-bit signed field at offset 4: -5
|
||||
let stored = (((-5i32) as u32) & 0x1_FFFF) << 4;
|
||||
assert_eq!(
|
||||
decode_int(&int(4, true, be, 4, 17), &stored.to_be_bytes()),
|
||||
Some(-5)
|
||||
);
|
||||
assert_eq!(
|
||||
decode_int(&int(4, true, DatatypeByteOrder::Vax, 0, 32), &[0; 4]),
|
||||
None
|
||||
);
|
||||
assert_eq!(
|
||||
decode_int(
|
||||
&int(4, true, DatatypeByteOrder::LittleEndian, 0, 32),
|
||||
&[0; 2]
|
||||
),
|
||||
None
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn floats_print_at_their_own_precision() {
|
||||
assert_eq!(fmt_float(f64::from(0.1f32), 32), "0.1");
|
||||
assert_eq!(fmt_float(0.1, 64), "0.1");
|
||||
assert_eq!(fmt_float(1e20, 64), "1e20");
|
||||
assert_eq!(fmt_float(2.0, 64), "2");
|
||||
assert_eq!(fmt_float(f64::NAN, 64), "NaN");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user