//! Decoding one element of any datatype into a [`Value`], and printing it. //! //! Decoding never panics: a short buffer, an unknown byte order or a //! dangling heap reference becomes [`Value::Error`]. use std::cell::RefCell; use clawhdf5_format::datatype::{Datatype, DatatypeByteOrder, ReferenceType, StringPadding}; use clawhdf5_format::vl_data::{VlResolver, check_element_size}; use serde_json::Value as J; use crate::dtype; use crate::h5::H5; #[derive(Debug, Clone, PartialEq)] pub enum Value { Int(i128), /// A float and the width (in bits) it was stored with, so it prints at /// its own precision. Float(f64, u8), Str(String), /// A null variable-length string (heap address 0): h5dump prints it as /// `NULL`, h5py reads it as empty. NullStr, /// Opaque, bitfield, time and oversized integers. Bytes(Vec), /// An enum member (name, when the value matches one) and its value. Enum(Option, i128), Compound(Vec<(String, Value)>), Array(Vec), /// A variable-length sequence. Seq(Vec), /// A reference: the referenced object's address (`None` = null). Ref(Option), /// A region or attribute reference, kept as its bytes. OtherRef(Vec), Error(String), } pub fn hex(b: &[u8]) -> String { let mut s = String::from("0x"); for x in b { s.push_str(&format!("{x:02x}")); } s } /// Element bytes as an unsigned integer (at most 16 bytes). fn bits(b: &[u8], order: &DatatypeByteOrder) -> Option { if b.len() > 16 { return None; } let mut v = 0u128; match order { DatatypeByteOrder::LittleEndian => { for (i, x) in b.iter().enumerate() { v |= u128::from(*x) << (8 * i); } } DatatypeByteOrder::BigEndian => { for x in b { v = (v << 8) | u128::from(*x); } } DatatypeByteOrder::Vax => return None, } Some(v) } /// The value of an integer (fixed-point) element, honouring its bit offset /// and precision. `None` when it cannot be represented (over 16 bytes) or /// `dt` is not an integer. pub fn decode_int(dt: &Datatype, b: &[u8]) -> Option { let Datatype::FixedPoint { size, byte_order, signed, bit_offset, bit_precision, } = dt else { return None; }; let size = usize::try_from(*size).ok()?; let v = bits(b.get(..size)?, byte_order)?; let off = u32::from(*bit_offset); let prec = u32::from(*bit_precision).min(128); if off >= 128 || prec == 0 { return Some(0); } let mut x = v >> off; if prec < 128 { x &= (1u128 << prec) - 1; } if *signed && prec < 128 && (x >> (prec - 1)) & 1 == 1 { x |= !0u128 << prec; return Some(x as i128); } if !*signed && prec == 128 && x > i128::MAX as u128 { return None; } Some(x as i128) } fn decode_float(dt: &Datatype, b: &[u8]) -> Value { let Datatype::FloatingPoint { size, byte_order, .. } = dt else { return Value::Error("not a float".into()); }; let Ok(n) = usize::try_from(*size) else { return Value::Error("float size".into()); }; let Some(b) = b.get(..n) else { return Value::Error("short element".into()); }; if dtype::is_ieee(dt) && let Some(v) = bits(b, byte_order) { return match n { 2 => Value::Float( f64::from(clawhdf5_format::float16::f16_bits_to_f32(v as u16)), 16, ), 4 => Value::Float(f64::from(f32::from_bits(v as u32)), 32), _ => Value::Float(f64::from_bits(v as u64), 64), }; } // Non-IEEE layouts (N-Bit floats, VAX order): the library converts. match clawhdf5_format::data_read::read_as_f64(b, dt) { Ok(v) if v.len() == 1 => Value::Float(v[0], (n * 8).min(64) as u8), Ok(_) => Value::Error("float conversion".into()), Err(e) => Value::Error(e.to_string()), } } fn trim_string(b: &[u8], pad: &StringPadding) -> String { let cut = b.iter().position(|&c| c == 0).unwrap_or(b.len()); let mut s = &b[..cut]; if matches!(pad, StringPadding::SpacePad) { while let [rest @ .., b' '] = s { s = rest; } } String::from_utf8_lossy(s).into_owned() } /// Decodes elements of one file. pub struct Decoder<'a> { pub h5: &'a H5, /// Variable-length elements are resolved as the library resolves them /// (so as libhdf5 does), not by a decoder of our own. vl: RefCell>, } impl<'a> Decoder<'a> { pub fn new(h5: &'a H5) -> Self { Self { h5, vl: RefCell::new(VlResolver::new(h5.data(), h5.os(), h5.ls())), } } /// Decode element `i` of `raw`, an array of `dt` elements. pub fn element(&self, dt: &Datatype, raw: &[u8], i: usize) -> Value { let size = dt.type_size() as usize; match i .checked_mul(size) .and_then(|s| raw.get(s..s.checked_add(size)?)) { Some(b) => self.decode(dt, b, 0), None => Value::Error("element out of range".into()), } } pub fn decode(&self, dt: &Datatype, b: &[u8], depth: u32) -> Value { if depth > 32 { return Value::Error("datatype nesting too deep".into()); } let size = dt.type_size() as usize; let Some(b) = b.get(..size) else { return Value::Error("short element".into()); }; match dt { Datatype::FixedPoint { .. } => match decode_int(dt, b) { Some(v) => Value::Int(v), None => Value::Bytes(b.to_vec()), }, Datatype::FloatingPoint { .. } => decode_float(dt, b), Datatype::Time { .. } | Datatype::BitField { .. } | Datatype::Opaque { .. } => { Value::Bytes(b.to_vec()) } Datatype::String { padding, .. } => Value::Str(trim_string(b, padding)), Datatype::Compound { members, .. } => { let mut out = Vec::with_capacity(members.len()); for m in members { let off = usize::try_from(m.byte_offset).unwrap_or(usize::MAX); let v = match b.get(off..) { Some(mb) => self.decode(&m.datatype, mb, depth + 1), None => Value::Error("member out of bounds".into()), }; out.push((m.name.clone(), v)); } Value::Compound(out) } Datatype::Reference { ref_type, .. } => match ref_type { ReferenceType::Object | ReferenceType::Object2 => { match clawhdf5_format::data_read::read_object_references(b, dt, self.h5.os()) { Ok(r) if r.len() == 1 => { let a = r[0].address; let undef = a == u64::MAX || (self.h5.os() < 8 && a == (1u64 << (8 * self.h5.os())) - 1); Value::Ref(if undef || a == 0 { None } else { Some(a) }) } Ok(_) => Value::Error("reference".into()), Err(e) => Value::Error(e.to_string()), } } _ => Value::OtherRef(b.to_vec()), }, Datatype::Enumeration { base_type, members, .. } => { let Some(v) = decode_int(base_type, b) else { return Value::Bytes(b.to_vec()); }; let bs = base_type.type_size() as usize; let name = members .iter() .find(|m| m.value.get(..bs) == b.get(..bs)) .map(|m| m.name.clone()); Value::Enum(name, v) } Datatype::Array { base_type, dimensions, } => { let n = dimensions .iter() .try_fold(1usize, |a, &d| a.checked_mul(d as usize)); let bs = base_type.type_size() as usize; let Some(n) = n.filter(|n| n.checked_mul(bs).is_some_and(|t| t <= b.len())) else { return Value::Error("array larger than its element".into()); }; let mut out = Vec::with_capacity(n); for k in 0..n { out.push(self.decode(base_type, &b[k * bs..], depth + 1)); } Value::Array(out) } Datatype::VariableLength { size, is_string, base_type, .. } => match check_element_size(*size, self.h5.os()) { Ok(()) => self.decode_vlen(*is_string, base_type, b, depth), Err(e) => Value::Error(e.to_string()), }, } } /// A variable-length element, resolved by the library's /// [`VlResolver`]: a string ends at its first NUL, a heap object whose /// size is not the element's length × base size is an error, and a /// heap address of 0 is null — all as libhdf5 (and so h5dump and h5py) /// has it. fn decode_vlen(&self, is_string: bool, base: &Datatype, b: &[u8], depth: u32) -> Value { if is_string { return match self.vl.borrow_mut().string_element(b) { Ok(Some(s)) => Value::Str(String::from_utf8_lossy(s).into_owned()), Ok(None) => Value::NullStr, Err(e) => Value::Error(e.to_string()), }; } let bs = base.type_size() as usize; if bs == 0 { return Value::Error("VL base type of size 0".into()); } let obj = match self.vl.borrow_mut().element(b, bs) { Ok(o) => o.unwrap_or(&[]), Err(e) => return Value::Error(e.to_string()), }; Value::Seq( obj.chunks_exact(bs) .map(|e| self.decode(base, e, depth + 1)) .collect(), ) } } /// Format a float like C's `%g` would at full round-trip precision: plain /// digits for moderate magnitudes, an exponent otherwise. pub fn fmt_float(v: f64, width: u8) -> String { if v.is_nan() { return "NaN".into(); } if v.is_infinite() { return if v > 0.0 { "Inf".into() } else { "-Inf".into() }; } let a = v.abs(); let plain = a == 0.0 || (1e-5..1e16).contains(&a); match (width, plain) { (16 | 32, true) => format!("{}", v as f32), (16 | 32, false) => format!("{:e}", v as f32), (_, true) => format!("{v}"), (_, false) => format!("{v:e}"), } } fn escape(s: &str) -> String { let mut o = String::with_capacity(s.len() + 2); for c in s.chars() { match c { '"' => o.push_str("\\\""), '\\' => o.push_str("\\\\"), '\n' => o.push_str("\\n"), '\r' => o.push_str("\\r"), '\t' => o.push_str("\\t"), c if (c as u32) < 0x20 => o.push_str(&format!("\\{:03o}", c as u32)), c => o.push(c), } } o } /// A fixed-length string element's bytes quoted as h5dump prints a /// null-padded string: every byte, NULs as `\000`. pub fn quote_bytes(b: &[u8]) -> String { format!("\"{}\"", escape(&String::from_utf8_lossy(b))) } /// Text form, as in an h5dump DATA block. pub fn text(v: &Value, h5paths: &dyn Fn(u64) -> Option) -> String { match v { Value::Int(i) => i.to_string(), Value::Float(f, w) => fmt_float(*f, *w), Value::Str(s) => format!("\"{}\"", escape(s)), Value::NullStr => "NULL".into(), Value::Bytes(b) => hex(b), Value::Enum(Some(n), _) => n.clone(), Value::Enum(None, i) => i.to_string(), Value::Compound(ms) => format!( "{{ {} }}", ms.iter() .map(|(_, v)| text(v, h5paths)) .collect::>() .join(", ") ), Value::Array(vs) => format!( "[ {} ]", vs.iter() .map(|v| text(v, h5paths)) .collect::>() .join(", ") ), Value::Seq(vs) => format!( "({})", vs.iter() .map(|v| text(v, h5paths)) .collect::>() .join(", ") ), Value::Ref(None) => "NULL".into(), Value::Ref(Some(a)) => match h5paths(*a) { Some(p) => format!("\"{p}\""), None => format!("{a:#x}"), }, Value::OtherRef(b) => hex(b), Value::Error(e) => format!(""), } } /// hdf5-json value form. pub fn to_json(v: &Value, h5paths: &dyn Fn(u64) -> Option) -> J { match v { Value::Int(i) => { if let Ok(x) = i64::try_from(*i) { J::from(x) } else if let Ok(x) = u64::try_from(*i) { J::from(x) } else { J::from(i.to_string()) } } Value::Float(f, _) => { if f.is_finite() { serde_json::Number::from_f64(*f) .map(J::Number) .unwrap_or(J::Null) } else if f.is_nan() { J::from("NaN") } else if *f > 0.0 { J::from("Infinity") } else { J::from("-Infinity") } } Value::Str(s) => J::from(s.as_str()), Value::NullStr => J::from(""), Value::Bytes(b) | Value::OtherRef(b) => J::from(hex(b)), Value::Enum(_, i) => to_json(&Value::Int(*i), h5paths), Value::Compound(ms) => J::Array(ms.iter().map(|(_, v)| to_json(v, h5paths)).collect()), Value::Array(vs) | Value::Seq(vs) => { J::Array(vs.iter().map(|v| to_json(v, h5paths)).collect()) } Value::Ref(None) => J::Null, Value::Ref(Some(a)) => J::from(h5paths(*a).unwrap_or_else(|| format!("{a:#x}"))), Value::Error(e) => serde_json::json!({ "error": e }), } } #[cfg(test)] mod tests { use super::*; fn int(size: u32, signed: bool, order: DatatypeByteOrder, off: u16, prec: u16) -> Datatype { Datatype::FixedPoint { size, byte_order: order, signed, bit_offset: off, bit_precision: prec, } } #[test] fn integers_decode_with_order_offset_and_sign() { let le = DatatypeByteOrder::LittleEndian; let be = DatatypeByteOrder::BigEndian; assert_eq!( decode_int(&int(2, true, le.clone(), 0, 16), &[0xff, 0xff]), Some(-1) ); assert_eq!( decode_int(&int(2, false, le, 0, 16), &[0xff, 0xff]), Some(65535) ); assert_eq!( decode_int(&int(2, true, be.clone(), 0, 16), &[0x80, 0x00]), Some(-32768) ); // 17-bit signed field at offset 4: -5 let stored = (((-5i32) as u32) & 0x1_FFFF) << 4; assert_eq!( decode_int(&int(4, true, be, 4, 17), &stored.to_be_bytes()), Some(-5) ); assert_eq!( decode_int(&int(4, true, DatatypeByteOrder::Vax, 0, 32), &[0; 4]), None ); assert_eq!( decode_int( &int(4, true, DatatypeByteOrder::LittleEndian, 0, 32), &[0; 2] ), None ); } #[test] fn floats_print_at_their_own_precision() { assert_eq!(fmt_float(f64::from(0.1f32), 32), "0.1"); assert_eq!(fmt_float(0.1, 64), "0.1"); assert_eq!(fmt_float(1e20, 64), "1e20"); assert_eq!(fmt_float(2.0, 64), "2"); assert_eq!(fmt_float(f64::NAN, 64), "NaN"); } }