osobhandClaude Opus 5.5
d0d5347cd9
- Datatype::Complex serializes class 11 version 5 byte-identically to
libhdf5 2.2.0; containers holding it are written as version 5.
- DatasetBuilder::with_complex_f32/f64_data (h5py's {r, i} compound,
default) and with_native_complex_f32/f64_data (class 11, opt-in);
make_(native_)complex_f32/f64_type for attributes.
- Dataset::read_complex_f64/f32 read either form.
- Python create_dataset accepts complex64/complex128 (compound form).
- Parsing unchanged: class 11 still surfaces as {r, i}.
- Tests vs h5py 3.16 / libhdf5 2.0.0 and h5dump 2.2.0; docs.
Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
478 lines
16 KiB
Rust
478 lines
16 KiB
Rust
//! Decoding one element of any datatype into a [`Value`], and printing it.
|
||
//!
|
||
//! Decoding never panics: a short buffer, an unknown byte order or a
|
||
//! dangling heap reference becomes [`Value::Error`].
|
||
|
||
use std::cell::RefCell;
|
||
|
||
use clawhdf5_format::datatype::{Datatype, DatatypeByteOrder, ReferenceType, StringPadding};
|
||
use clawhdf5_format::vl_data::{VlResolver, check_element_size};
|
||
use serde_json::Value as J;
|
||
|
||
use crate::dtype;
|
||
use crate::h5::H5;
|
||
|
||
#[derive(Debug, Clone, PartialEq)]
|
||
pub enum Value {
|
||
Int(i128),
|
||
/// A float and the width (in bits) it was stored with, so it prints at
|
||
/// its own precision.
|
||
Float(f64, u8),
|
||
Str(String),
|
||
/// A null variable-length string (heap address 0): h5dump prints it as
|
||
/// `NULL`, h5py reads it as empty.
|
||
NullStr,
|
||
/// Opaque, bitfield, time and oversized integers.
|
||
Bytes(Vec<u8>),
|
||
/// An enum member (name, when the value matches one) and its value.
|
||
Enum(Option<String>, i128),
|
||
Compound(Vec<(String, Value)>),
|
||
Array(Vec<Value>),
|
||
/// A variable-length sequence.
|
||
Seq(Vec<Value>),
|
||
/// A reference: the referenced object's address (`None` = null).
|
||
Ref(Option<u64>),
|
||
/// A region or attribute reference, kept as its bytes.
|
||
OtherRef(Vec<u8>),
|
||
Error(String),
|
||
}
|
||
|
||
pub fn hex(b: &[u8]) -> String {
|
||
let mut s = String::from("0x");
|
||
for x in b {
|
||
s.push_str(&format!("{x:02x}"));
|
||
}
|
||
s
|
||
}
|
||
|
||
/// Element bytes as an unsigned integer (at most 16 bytes).
|
||
fn bits(b: &[u8], order: &DatatypeByteOrder) -> Option<u128> {
|
||
if b.len() > 16 {
|
||
return None;
|
||
}
|
||
let mut v = 0u128;
|
||
match order {
|
||
DatatypeByteOrder::LittleEndian => {
|
||
for (i, x) in b.iter().enumerate() {
|
||
v |= u128::from(*x) << (8 * i);
|
||
}
|
||
}
|
||
DatatypeByteOrder::BigEndian => {
|
||
for x in b {
|
||
v = (v << 8) | u128::from(*x);
|
||
}
|
||
}
|
||
DatatypeByteOrder::Vax => return None,
|
||
}
|
||
Some(v)
|
||
}
|
||
|
||
/// The value of an integer (fixed-point) element, honouring its bit offset
|
||
/// and precision. `None` when it cannot be represented (over 16 bytes) or
|
||
/// `dt` is not an integer.
|
||
pub fn decode_int(dt: &Datatype, b: &[u8]) -> Option<i128> {
|
||
let Datatype::FixedPoint {
|
||
size,
|
||
byte_order,
|
||
signed,
|
||
bit_offset,
|
||
bit_precision,
|
||
} = dt
|
||
else {
|
||
return None;
|
||
};
|
||
let size = usize::try_from(*size).ok()?;
|
||
let v = bits(b.get(..size)?, byte_order)?;
|
||
let off = u32::from(*bit_offset);
|
||
let prec = u32::from(*bit_precision).min(128);
|
||
if off >= 128 || prec == 0 {
|
||
return Some(0);
|
||
}
|
||
let mut x = v >> off;
|
||
if prec < 128 {
|
||
x &= (1u128 << prec) - 1;
|
||
}
|
||
if *signed && prec < 128 && (x >> (prec - 1)) & 1 == 1 {
|
||
x |= !0u128 << prec;
|
||
return Some(x as i128);
|
||
}
|
||
if !*signed && prec == 128 && x > i128::MAX as u128 {
|
||
return None;
|
||
}
|
||
Some(x as i128)
|
||
}
|
||
|
||
fn decode_float(dt: &Datatype, b: &[u8]) -> Value {
|
||
let Datatype::FloatingPoint {
|
||
size, byte_order, ..
|
||
} = dt
|
||
else {
|
||
return Value::Error("not a float".into());
|
||
};
|
||
let Ok(n) = usize::try_from(*size) else {
|
||
return Value::Error("float size".into());
|
||
};
|
||
let Some(b) = b.get(..n) else {
|
||
return Value::Error("short element".into());
|
||
};
|
||
if dtype::is_ieee(dt)
|
||
&& let Some(v) = bits(b, byte_order)
|
||
{
|
||
return match n {
|
||
2 => Value::Float(
|
||
f64::from(clawhdf5_format::float16::f16_bits_to_f32(v as u16)),
|
||
16,
|
||
),
|
||
4 => Value::Float(f64::from(f32::from_bits(v as u32)), 32),
|
||
_ => Value::Float(f64::from_bits(v as u64), 64),
|
||
};
|
||
}
|
||
// Non-IEEE layouts (N-Bit floats, VAX order): the library converts.
|
||
match clawhdf5_format::data_read::read_as_f64(b, dt) {
|
||
Ok(v) if v.len() == 1 => Value::Float(v[0], (n * 8).min(64) as u8),
|
||
Ok(_) => Value::Error("float conversion".into()),
|
||
Err(e) => Value::Error(e.to_string()),
|
||
}
|
||
}
|
||
|
||
fn trim_string(b: &[u8], pad: &StringPadding) -> String {
|
||
let cut = b.iter().position(|&c| c == 0).unwrap_or(b.len());
|
||
let mut s = &b[..cut];
|
||
if matches!(pad, StringPadding::SpacePad) {
|
||
while let [rest @ .., b' '] = s {
|
||
s = rest;
|
||
}
|
||
}
|
||
String::from_utf8_lossy(s).into_owned()
|
||
}
|
||
|
||
/// Decodes elements of one file.
|
||
pub struct Decoder<'a> {
|
||
pub h5: &'a H5,
|
||
/// Variable-length elements are resolved as the library resolves them
|
||
/// (so as libhdf5 does), not by a decoder of our own.
|
||
vl: RefCell<VlResolver<'a, dyn clawhdf5_format::storage::Storage + 'a>>,
|
||
}
|
||
|
||
impl<'a> Decoder<'a> {
|
||
pub fn new(h5: &'a H5) -> Self {
|
||
Self {
|
||
h5,
|
||
vl: RefCell::new(VlResolver::new_in(h5.store(), h5.os(), h5.ls())),
|
||
}
|
||
}
|
||
|
||
/// Decode element `i` of `raw`, an array of `dt` elements.
|
||
pub fn element(&self, dt: &Datatype, raw: &[u8], i: usize) -> Value {
|
||
let size = dt.type_size() as usize;
|
||
match i
|
||
.checked_mul(size)
|
||
.and_then(|s| raw.get(s..s.checked_add(size)?))
|
||
{
|
||
Some(b) => self.decode(dt, b, 0),
|
||
None => Value::Error("element out of range".into()),
|
||
}
|
||
}
|
||
|
||
pub fn decode(&self, dt: &Datatype, b: &[u8], depth: u32) -> Value {
|
||
if depth > 32 {
|
||
return Value::Error("datatype nesting too deep".into());
|
||
}
|
||
let size = dt.type_size() as usize;
|
||
let Some(b) = b.get(..size) else {
|
||
return Value::Error("short element".into());
|
||
};
|
||
match dt {
|
||
Datatype::Complex { size, base_type } => self.decode(
|
||
&Datatype::complex_as_compound(*size, base_type),
|
||
b,
|
||
depth + 1,
|
||
),
|
||
Datatype::FixedPoint { .. } => match decode_int(dt, b) {
|
||
Some(v) => Value::Int(v),
|
||
None => Value::Bytes(b.to_vec()),
|
||
},
|
||
Datatype::FloatingPoint { .. } => decode_float(dt, b),
|
||
Datatype::Time { .. } | Datatype::BitField { .. } | Datatype::Opaque { .. } => {
|
||
Value::Bytes(b.to_vec())
|
||
}
|
||
Datatype::String { padding, .. } => Value::Str(trim_string(b, padding)),
|
||
Datatype::Compound { members, .. } => {
|
||
let mut out = Vec::with_capacity(members.len());
|
||
for m in members {
|
||
let off = usize::try_from(m.byte_offset).unwrap_or(usize::MAX);
|
||
let v = match b.get(off..) {
|
||
Some(mb) => self.decode(&m.datatype, mb, depth + 1),
|
||
None => Value::Error("member out of bounds".into()),
|
||
};
|
||
out.push((m.name.clone(), v));
|
||
}
|
||
Value::Compound(out)
|
||
}
|
||
Datatype::Reference { ref_type, .. } => match ref_type {
|
||
ReferenceType::Object | ReferenceType::Object2 => {
|
||
match clawhdf5_format::data_read::read_object_references(b, dt, self.h5.os()) {
|
||
Ok(r) if r.len() == 1 => {
|
||
let a = r[0].address;
|
||
let undef = a == u64::MAX
|
||
|| (self.h5.os() < 8 && a == (1u64 << (8 * self.h5.os())) - 1);
|
||
Value::Ref(if undef || a == 0 { None } else { Some(a) })
|
||
}
|
||
Ok(_) => Value::Error("reference".into()),
|
||
Err(e) => Value::Error(e.to_string()),
|
||
}
|
||
}
|
||
_ => Value::OtherRef(b.to_vec()),
|
||
},
|
||
Datatype::Enumeration {
|
||
base_type, members, ..
|
||
} => {
|
||
let Some(v) = decode_int(base_type, b) else {
|
||
return Value::Bytes(b.to_vec());
|
||
};
|
||
let bs = base_type.type_size() as usize;
|
||
let name = members
|
||
.iter()
|
||
.find(|m| m.value.get(..bs) == b.get(..bs))
|
||
.map(|m| m.name.clone());
|
||
Value::Enum(name, v)
|
||
}
|
||
Datatype::Array {
|
||
base_type,
|
||
dimensions,
|
||
} => {
|
||
let n = dimensions
|
||
.iter()
|
||
.try_fold(1usize, |a, &d| a.checked_mul(d as usize));
|
||
let bs = base_type.type_size() as usize;
|
||
let Some(n) = n.filter(|n| n.checked_mul(bs).is_some_and(|t| t <= b.len())) else {
|
||
return Value::Error("array larger than its element".into());
|
||
};
|
||
let mut out = Vec::with_capacity(n);
|
||
for k in 0..n {
|
||
out.push(self.decode(base_type, &b[k * bs..], depth + 1));
|
||
}
|
||
Value::Array(out)
|
||
}
|
||
Datatype::VariableLength {
|
||
size,
|
||
is_string,
|
||
base_type,
|
||
..
|
||
} => match check_element_size(*size, self.h5.os()) {
|
||
Ok(()) => self.decode_vlen(*is_string, base_type, b, depth),
|
||
Err(e) => Value::Error(e.to_string()),
|
||
},
|
||
}
|
||
}
|
||
|
||
/// A variable-length element, resolved by the library's
|
||
/// [`VlResolver`]: a string ends at its first NUL, a heap object whose
|
||
/// size is not the element's length × base size is an error, and a
|
||
/// heap address of 0 is null — all as libhdf5 (and so h5dump and h5py)
|
||
/// has it.
|
||
fn decode_vlen(&self, is_string: bool, base: &Datatype, b: &[u8], depth: u32) -> Value {
|
||
if is_string {
|
||
return match self.vl.borrow_mut().string_element_in(b) {
|
||
Ok(Some(s)) => Value::Str(String::from_utf8_lossy(s).into_owned()),
|
||
Ok(None) => Value::NullStr,
|
||
Err(e) => Value::Error(e.to_string()),
|
||
};
|
||
}
|
||
let bs = base.type_size() as usize;
|
||
if bs == 0 {
|
||
return Value::Error("VL base type of size 0".into());
|
||
}
|
||
// Copied out: decoding an element may resolve nested ones.
|
||
let obj = match self.vl.borrow_mut().element_in(b, bs) {
|
||
Ok(o) => o.unwrap_or(&[]).to_vec(),
|
||
Err(e) => return Value::Error(e.to_string()),
|
||
};
|
||
Value::Seq(
|
||
obj.chunks_exact(bs)
|
||
.map(|e| self.decode(base, e, depth + 1))
|
||
.collect(),
|
||
)
|
||
}
|
||
}
|
||
|
||
/// Format a float like C's `%g` would at full round-trip precision: plain
|
||
/// digits for moderate magnitudes, an exponent otherwise.
|
||
pub fn fmt_float(v: f64, width: u8) -> String {
|
||
if v.is_nan() {
|
||
return "NaN".into();
|
||
}
|
||
if v.is_infinite() {
|
||
return if v > 0.0 { "Inf".into() } else { "-Inf".into() };
|
||
}
|
||
let a = v.abs();
|
||
let plain = a == 0.0 || (1e-5..1e16).contains(&a);
|
||
match (width, plain) {
|
||
(16 | 32, true) => format!("{}", v as f32),
|
||
(16 | 32, false) => format!("{:e}", v as f32),
|
||
(_, true) => format!("{v}"),
|
||
(_, false) => format!("{v:e}"),
|
||
}
|
||
}
|
||
|
||
fn escape(s: &str) -> String {
|
||
let mut o = String::with_capacity(s.len() + 2);
|
||
for c in s.chars() {
|
||
match c {
|
||
'"' => o.push_str("\\\""),
|
||
'\\' => o.push_str("\\\\"),
|
||
'\n' => o.push_str("\\n"),
|
||
'\r' => o.push_str("\\r"),
|
||
'\t' => o.push_str("\\t"),
|
||
c if (c as u32) < 0x20 => o.push_str(&format!("\\{:03o}", c as u32)),
|
||
c => o.push(c),
|
||
}
|
||
}
|
||
o
|
||
}
|
||
|
||
/// A fixed-length string element's bytes quoted as h5dump prints a
|
||
/// null-padded string: every byte, NULs as `\000`.
|
||
pub fn quote_bytes(b: &[u8]) -> String {
|
||
format!("\"{}\"", escape(&String::from_utf8_lossy(b)))
|
||
}
|
||
|
||
/// Text form, as in an h5dump DATA block.
|
||
pub fn text(v: &Value, h5paths: &dyn Fn(u64) -> Option<String>) -> String {
|
||
match v {
|
||
Value::Int(i) => i.to_string(),
|
||
Value::Float(f, w) => fmt_float(*f, *w),
|
||
Value::Str(s) => format!("\"{}\"", escape(s)),
|
||
Value::NullStr => "NULL".into(),
|
||
Value::Bytes(b) => hex(b),
|
||
Value::Enum(Some(n), _) => n.clone(),
|
||
Value::Enum(None, i) => i.to_string(),
|
||
Value::Compound(ms) => format!(
|
||
"{{ {} }}",
|
||
ms.iter()
|
||
.map(|(_, v)| text(v, h5paths))
|
||
.collect::<Vec<_>>()
|
||
.join(", ")
|
||
),
|
||
Value::Array(vs) => format!(
|
||
"[ {} ]",
|
||
vs.iter()
|
||
.map(|v| text(v, h5paths))
|
||
.collect::<Vec<_>>()
|
||
.join(", ")
|
||
),
|
||
Value::Seq(vs) => format!(
|
||
"({})",
|
||
vs.iter()
|
||
.map(|v| text(v, h5paths))
|
||
.collect::<Vec<_>>()
|
||
.join(", ")
|
||
),
|
||
Value::Ref(None) => "NULL".into(),
|
||
Value::Ref(Some(a)) => match h5paths(*a) {
|
||
Some(p) => format!("\"{p}\""),
|
||
None => format!("{a:#x}"),
|
||
},
|
||
Value::OtherRef(b) => hex(b),
|
||
Value::Error(e) => format!("<error: {e}>"),
|
||
}
|
||
}
|
||
|
||
/// hdf5-json value form.
|
||
pub fn to_json(v: &Value, h5paths: &dyn Fn(u64) -> Option<String>) -> J {
|
||
match v {
|
||
Value::Int(i) => {
|
||
if let Ok(x) = i64::try_from(*i) {
|
||
J::from(x)
|
||
} else if let Ok(x) = u64::try_from(*i) {
|
||
J::from(x)
|
||
} else {
|
||
J::from(i.to_string())
|
||
}
|
||
}
|
||
Value::Float(f, _) => {
|
||
if f.is_finite() {
|
||
serde_json::Number::from_f64(*f)
|
||
.map(J::Number)
|
||
.unwrap_or(J::Null)
|
||
} else if f.is_nan() {
|
||
J::from("NaN")
|
||
} else if *f > 0.0 {
|
||
J::from("Infinity")
|
||
} else {
|
||
J::from("-Infinity")
|
||
}
|
||
}
|
||
Value::Str(s) => J::from(s.as_str()),
|
||
Value::NullStr => J::from(""),
|
||
Value::Bytes(b) | Value::OtherRef(b) => J::from(hex(b)),
|
||
Value::Enum(_, i) => to_json(&Value::Int(*i), h5paths),
|
||
Value::Compound(ms) => J::Array(ms.iter().map(|(_, v)| to_json(v, h5paths)).collect()),
|
||
Value::Array(vs) | Value::Seq(vs) => {
|
||
J::Array(vs.iter().map(|v| to_json(v, h5paths)).collect())
|
||
}
|
||
Value::Ref(None) => J::Null,
|
||
Value::Ref(Some(a)) => J::from(h5paths(*a).unwrap_or_else(|| format!("{a:#x}"))),
|
||
Value::Error(e) => serde_json::json!({ "error": e }),
|
||
}
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
|
||
fn int(size: u32, signed: bool, order: DatatypeByteOrder, off: u16, prec: u16) -> Datatype {
|
||
Datatype::FixedPoint {
|
||
size,
|
||
byte_order: order,
|
||
signed,
|
||
bit_offset: off,
|
||
bit_precision: prec,
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn integers_decode_with_order_offset_and_sign() {
|
||
let le = DatatypeByteOrder::LittleEndian;
|
||
let be = DatatypeByteOrder::BigEndian;
|
||
assert_eq!(
|
||
decode_int(&int(2, true, le.clone(), 0, 16), &[0xff, 0xff]),
|
||
Some(-1)
|
||
);
|
||
assert_eq!(
|
||
decode_int(&int(2, false, le, 0, 16), &[0xff, 0xff]),
|
||
Some(65535)
|
||
);
|
||
assert_eq!(
|
||
decode_int(&int(2, true, be.clone(), 0, 16), &[0x80, 0x00]),
|
||
Some(-32768)
|
||
);
|
||
// 17-bit signed field at offset 4: -5
|
||
let stored = (((-5i32) as u32) & 0x1_FFFF) << 4;
|
||
assert_eq!(
|
||
decode_int(&int(4, true, be, 4, 17), &stored.to_be_bytes()),
|
||
Some(-5)
|
||
);
|
||
assert_eq!(
|
||
decode_int(&int(4, true, DatatypeByteOrder::Vax, 0, 32), &[0; 4]),
|
||
None
|
||
);
|
||
assert_eq!(
|
||
decode_int(
|
||
&int(4, true, DatatypeByteOrder::LittleEndian, 0, 32),
|
||
&[0; 2]
|
||
),
|
||
None
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn floats_print_at_their_own_precision() {
|
||
assert_eq!(fmt_float(f64::from(0.1f32), 32), "0.1");
|
||
assert_eq!(fmt_float(0.1, 64), "0.1");
|
||
assert_eq!(fmt_float(1e20, 64), "1e20");
|
||
assert_eq!(fmt_float(2.0, 64), "2");
|
||
assert_eq!(fmt_float(f64::NAN, 64), "NaN");
|
||
}
|
||
}
|