quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases 1 Wiki Activity

Merge branch 'fix/p0-reader-numeric' into fix/phase0-correctness

Browse Source
This commit is contained in:
osobh
2026-09-25 21:20:59 -05:00
parent 36d7a6f234 2f252df084
commit e1115bc92a
3 changed files with 741 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/data_read.rs
+357 -93
View File
@@ -773,14 +773,7 @@ pub fn read_as_f64_zerocopy<'a>(raw: &'a [u8], datatype: &Datatype) -> Option<&'
// Only native LE f64 is eligible
#[cfg(target_endian = "little")]
{
if !matches!(
datatype,
Datatype::FloatingPoint {
size: 8,
byte_order: DatatypeByteOrder::LittleEndian,
..
}
) {
if !is_native_le_float(datatype, FloatFormat::Double) {
return None;
}
if !raw.len().is_multiple_of(8) {
@@ -809,14 +802,7 @@ pub fn read_as_f64_zerocopy<'a>(raw: &'a [u8], datatype: &Datatype) -> Option<&'
pub fn read_as_f32_zerocopy<'a>(raw: &'a [u8], datatype: &Datatype) -> Option<&'a [f32]> {
#[cfg(target_endian = "little")]
{
if !matches!(
datatype,
Datatype::FloatingPoint {
size: 4,
byte_order: DatatypeByteOrder::LittleEndian,
..
}
) {
if !is_native_le_float(datatype, FloatFormat::Single) {
return None;
}
if !raw.len().is_multiple_of(4) {
@@ -902,9 +888,9 @@ fn native_le_to_vec<T: Copy>(raw: &[u8], count: usize) -> Vec<T> {
/// Convert raw bytes to `f64` values.
pub fn read_as_f64(raw: &[u8], datatype: &Datatype) -> Result<Vec<f64>, FormatError> {
// Array datatypes (e.g. an array-typed compound member) are read as a flat
// sequence of their base elements.
if let Datatype::Array { base_type, .. } = datatype {
// Array datatypes read as a flat sequence of their base elements, and
// enumerations (h5py's bool among them) as their integer values.
if let Datatype::Array { base_type, .. } | Datatype::Enumeration { base_type, .. } = datatype {
return read_as_f64(raw, base_type);
}
ensure_numeric(datatype, "FloatingPoint or FixedPoint")?;
@@ -919,20 +905,19 @@ pub fn read_as_f64(raw: &[u8], datatype: &Datatype) -> Result<Vec<f64>, FormatEr
// Fast path: native-endian f64 — single bulk memcpy
#[cfg(target_endian = "little")]
if matches!(
datatype,
Datatype::FloatingPoint {
size: 8,
byte_order: DatatypeByteOrder::LittleEndian,
..
}
) {
if is_native_le_float(datatype, FloatFormat::Double) {
return Ok(native_le_to_vec::<f64>(raw, count));
}
let order = get_byte_order(datatype);
let mut result = Vec::with_capacity(count);
if let Datatype::FloatingPoint { .. } = datatype {
let format = FloatFormat::of(datatype)?;
for chunk in raw.chunks_exact(elem_size) {
result.push(format.decode(chunk, &order));
}
return Ok(result);
}
for i in 0..count {
let chunk = &raw[i * elem_size..(i + 1) * elem_size];
let val = convert_to_f64(chunk, datatype, &order)?;
@@ -947,18 +932,7 @@ fn convert_to_f64(
order: &DatatypeByteOrder,
) -> Result<f64, FormatError> {
match dt {
Datatype::FloatingPoint { size, .. } => match size {
4 => {
let v = read_f32_bytes(bytes, order);
Ok(v as f64)
}
8 => Ok(read_f64_bytes(bytes, order)),
2 => Ok(read_f16_bytes(bytes, order) as f64),
_ => Err(FormatError::DataSizeMismatch {
expected: 8,
actual: *size as usize,
}),
},
Datatype::FloatingPoint { .. } => Ok(FloatFormat::of(dt)?.decode(bytes, order)),
Datatype::FixedPoint {
size,
signed,
@@ -982,9 +956,83 @@ fn convert_to_f64(
}
}
/// One numeric element as stored, before conversion to the caller's type.
#[derive(Debug, Clone, Copy, PartialEq)]
enum Scalar {
Signed(i64),
Unsigned(u64),
Float(f64),
}
impl Scalar {
// Every conversion follows libhdf5's default (hard) conversions: a value
// outside the target type's range saturates to its minimum or maximum —
// including a negative value read as unsigned, which reads as 0 — rather
// than being truncated to its low bits. Floats truncate toward zero; NaN
// converts to 0 (libhdf5 leaves that case to the C cast, whose result is
// platform-dependent).
fn to_i64(self) -> i64 {
match self {
Scalar::Signed(v) => v,
Scalar::Unsigned(v) => i64::try_from(v).unwrap_or(i64::MAX),
Scalar::Float(v) => v as i64,
}
}
fn to_u64(self) -> u64 {
match self {
Scalar::Signed(v) => u64::try_from(v).unwrap_or(0),
Scalar::Unsigned(v) => v,
Scalar::Float(v) => v as u64,
}
}
fn to_i32(self) -> i32 {
match self {
Scalar::Signed(v) => v.clamp(i32::MIN.into(), i32::MAX.into()) as i32,
Scalar::Unsigned(v) => i32::try_from(v).unwrap_or(i32::MAX),
Scalar::Float(v) => v as i32,
}
}
}
/// Decode one element of a numeric datatype.
fn decode_scalar(
bytes: &[u8],
dt: &Datatype,
order: &DatatypeByteOrder,
) -> Result<Scalar, FormatError> {
match dt {
Datatype::FixedPoint {
size,
signed,
bit_offset,
bit_precision,
..
} => {
let full = read_unsigned_int(bytes, *size as usize, order);
let (off, prec) = effective_bits(*size as usize, *bit_offset, *bit_precision);
Ok(if *signed {
Scalar::Signed(extract_signed(full, off, prec))
} else {
Scalar::Unsigned(extract_unsigned(full, off, prec))
})
}
_ => convert_to_f64(bytes, dt, order).map(Scalar::Float),
}
}
/// Convert raw bytes to `i64` values.
///
/// Values are converted the way libhdf5 converts them: integers outside the
/// target range saturate at its minimum or maximum (a negative value read as
/// unsigned is 0), and floating-point data is truncated toward zero and
/// saturated, with NaN read as 0.
pub fn read_as_i64(raw: &[u8], datatype: &Datatype) -> Result<Vec<i64>, FormatError> {
if let Datatype::Array { base_type, .. } = datatype {
// Array datatypes read as a flat sequence of their base elements, and
// enumerations (h5py's bool among them) as their integer values.
if let Datatype::Array { base_type, .. } | Datatype::Enumeration { base_type, .. } = datatype {
return read_as_i64(raw, base_type);
}
ensure_numeric(datatype, "FixedPoint (signed)")?;
@@ -1014,19 +1062,24 @@ pub fn read_as_i64(raw: &[u8], datatype: &Datatype) -> Result<Vec<i64>, FormatEr
}
let order = get_byte_order(datatype);
let (off, prec) = fixed_bits(datatype);
let mut result = Vec::with_capacity(count);
for i in 0..count {
let chunk = &raw[i * elem_size..(i + 1) * elem_size];
let full = read_unsigned_int(chunk, elem_size, &order);
result.push(extract_signed(full, off, prec));
result.push(decode_scalar(chunk, datatype, &order)?.to_i64());
}
Ok(result)
}
/// Convert raw bytes to `u64` values.
///
/// Values are converted the way libhdf5 converts them: integers outside the
/// target range saturate at its minimum or maximum (a negative value read as
/// unsigned is 0), and floating-point data is truncated toward zero and
/// saturated, with NaN read as 0.
pub fn read_as_u64(raw: &[u8], datatype: &Datatype) -> Result<Vec<u64>, FormatError> {
if let Datatype::Array { base_type, .. } = datatype {
// Array datatypes read as a flat sequence of their base elements, and
// enumerations (h5py's bool among them) as their integer values.
if let Datatype::Array { base_type, .. } | Datatype::Enumeration { base_type, .. } = datatype {
return read_as_u64(raw, base_type);
}
ensure_numeric(datatype, "FixedPoint (unsigned)")?;
@@ -1039,19 +1092,19 @@ pub fn read_as_u64(raw: &[u8], datatype: &Datatype) -> Result<Vec<u64>, FormatEr
}
let count = raw.len() / elem_size;
let order = get_byte_order(datatype);
let (off, prec) = fixed_bits(datatype);
let mut result = Vec::with_capacity(count);
for i in 0..count {
let chunk = &raw[i * elem_size..(i + 1) * elem_size];
let full = read_unsigned_int(chunk, elem_size, &order);
result.push(extract_unsigned(full, off, prec));
result.push(decode_scalar(chunk, datatype, &order)?.to_u64());
}
Ok(result)
}
/// Convert raw bytes to `f32` values.
pub fn read_as_f32(raw: &[u8], datatype: &Datatype) -> Result<Vec<f32>, FormatError> {
if let Datatype::Array { base_type, .. } = datatype {
// Array datatypes read as a flat sequence of their base elements, and
// enumerations (h5py's bool among them) as their integer values.
if let Datatype::Array { base_type, .. } | Datatype::Enumeration { base_type, .. } = datatype {
return read_as_f32(raw, base_type);
}
ensure_numeric(datatype, "FloatingPoint")?;
@@ -1066,25 +1119,11 @@ pub fn read_as_f32(raw: &[u8], datatype: &Datatype) -> Result<Vec<f32>, FormatEr
// Fast path: native-endian f32 — single bulk memcpy
#[cfg(target_endian = "little")]
if matches!(
datatype,
Datatype::FloatingPoint {
size: 4,
byte_order: DatatypeByteOrder::LittleEndian,
..
}
) {
if is_native_le_float(datatype, FloatFormat::Single) {
return Ok(native_le_to_vec::<f32>(raw, count));
}
// Little-endian half precision (numpy float16): widen directly.
if matches!(
datatype,
Datatype::FloatingPoint {
size: 2,
byte_order: DatatypeByteOrder::LittleEndian,
..
}
) {
// Little-endian IEEE half precision (numpy float16): widen directly.
if is_native_le_float(datatype, FloatFormat::Half) {
let (halves, _) = raw[..count * 2].as_chunks::<2>();
return Ok(halves
.iter()
@@ -1094,18 +1133,22 @@ pub fn read_as_f32(raw: &[u8], datatype: &Datatype) -> Result<Vec<f32>, FormatEr
let order = get_byte_order(datatype);
let mut result = Vec::with_capacity(count);
if let Datatype::FloatingPoint { .. } = datatype {
let format = FloatFormat::of(datatype)?;
for chunk in raw.chunks_exact(elem_size) {
result.push(match format {
FloatFormat::Single => read_f32_bytes(chunk, &order),
FloatFormat::Half => read_f16_bytes(chunk, &order),
// Double rounds; every other supported layout (bfloat16, FP8)
// is exact in f32.
_ => format.decode(chunk, &order) as f32,
});
}
return Ok(result);
}
for i in 0..count {
let chunk = &raw[i * elem_size..(i + 1) * elem_size];
match datatype {
Datatype::FloatingPoint { size: 4, .. } => {
result.push(read_f32_bytes(chunk, &order));
}
Datatype::FloatingPoint { size: 8, .. } => {
result.push(read_f64_bytes(chunk, &order) as f32);
}
Datatype::FloatingPoint { size: 2, .. } => {
result.push(read_f16_bytes(chunk, &order));
}
Datatype::FixedPoint {
signed: true,
size,
@@ -1140,8 +1183,15 @@ pub fn read_as_f32(raw: &[u8], datatype: &Datatype) -> Result<Vec<f32>, FormatEr
}
/// Convert raw bytes to `i32` values.
///
/// Values are converted the way libhdf5 converts them: integers outside the
/// target range saturate at its minimum or maximum (a negative value read as
/// unsigned is 0), and floating-point data is truncated toward zero and
/// saturated, with NaN read as 0.
pub fn read_as_i32(raw: &[u8], datatype: &Datatype) -> Result<Vec<i32>, FormatError> {
if let Datatype::Array { base_type, .. } = datatype {
// Array datatypes read as a flat sequence of their base elements, and
// enumerations (h5py's bool among them) as their integer values.
if let Datatype::Array { base_type, .. } | Datatype::Enumeration { base_type, .. } = datatype {
return read_as_i32(raw, base_type);
}
ensure_numeric(datatype, "FixedPoint")?;
@@ -1162,6 +1212,7 @@ pub fn read_as_i32(raw: &[u8], datatype: &Datatype) -> Result<Vec<i32>, FormatEr
datatype,
Datatype::FixedPoint {
byte_order: DatatypeByteOrder::LittleEndian,
signed: true,
..
}
)
@@ -1170,12 +1221,10 @@ pub fn read_as_i32(raw: &[u8], datatype: &Datatype) -> Result<Vec<i32>, FormatEr
}
let order = get_byte_order(datatype);
let (off, prec) = fixed_bits(datatype);
let mut result = Vec::with_capacity(count);
for i in 0..count {
let chunk = &raw[i * elem_size..(i + 1) * elem_size];
let full = read_unsigned_int(chunk, elem_size, &order);
result.push(extract_signed(full, off, prec) as i32);
result.push(decode_scalar(chunk, datatype, &order)?.to_i32());
}
Ok(result)
}
@@ -1616,6 +1665,174 @@ fn reorder_bytes(bytes: &[u8], order: &DatatypeByteOrder) -> [u8; 8] {
buf
}
/// How the bits of a floating-point datatype are laid out, read from the
/// datatype message's fields rather than assumed from its size (a 2-byte
/// float may be IEEE half or bfloat16).
#[derive(Debug, Clone, Copy, PartialEq)]
enum FloatFormat {
/// IEEE-754 binary16.
Half,
/// IEEE-754 binary32.
Single,
/// IEEE-754 binary64.
Double,
/// Any other IEEE-style layout (implied leading mantissa bit, all-ones
/// exponent for infinity/NaN) whose values are all exact in `f64`:
/// bfloat16, the FP8 formats, and similar.
Other(FloatLayout),
}
#[derive(Debug, Clone, Copy, PartialEq)]
struct FloatLayout {
exponent_location: u32,
exponent_size: u32,
mantissa_location: u32,
mantissa_size: u32,
exponent_bias: u32,
}
impl FloatFormat {
fn of(dt: &Datatype) -> Result<FloatFormat, FormatError> {
let Datatype::FloatingPoint {
size,
exponent_location,
exponent_size,
mantissa_location,
mantissa_size,
exponent_bias,
..
} = dt
else {
return Err(FormatError::TypeMismatch {
expected: "FloatingPoint",
actual: datatype_name(dt),
});
};
let layout = FloatLayout {
exponent_location: u32::from(*exponent_location),
exponent_size: u32::from(*exponent_size),
mantissa_location: u32::from(*mantissa_location),
mantissa_size: u32::from(*mantissa_size),
exponent_bias: *exponent_bias,
};
let fields = (
layout.exponent_location,
layout.exponent_size,
layout.mantissa_location,
layout.mantissa_size,
layout.exponent_bias,
);
let bits = size.saturating_mul(8);
// The sign bit is not kept in `Datatype`; every standard layout has it
// directly above the exponent, with the mantissa below.
let well_formed = layout.exponent_size > 0
&& layout.mantissa_size > 0
&& layout.mantissa_location + layout.mantissa_size <= layout.exponent_location
&& layout.exponent_location + layout.exponent_size < bits;
match (size, fields) {
(2, (10, 5, 0, 10, 15)) => Ok(FloatFormat::Half),
(4, (23, 8, 0, 23, 127)) => Ok(FloatFormat::Single),
(8, (52, 11, 0, 52, 1023)) => Ok(FloatFormat::Double),
_ if well_formed
&& *size <= 8
&& layout.exponent_size <= 11
&& layout.mantissa_size <= 52 =>
{
Ok(FloatFormat::Other(layout))
}
// Fields that cannot describe any float (e.g. left zeroed by a
// hand-built datatype): fall back to the IEEE type of that size.
(2, _) if !well_formed => Ok(FloatFormat::Half),
(4, _) if !well_formed => Ok(FloatFormat::Single),
(8, _) if !well_formed => Ok(FloatFormat::Double),
// x87 80-bit extended, binary128, ...: not representable in f64.
_ => Err(FormatError::TypeMismatch {
expected: "floating point of at most 64 bits (IEEE-style layout)",
actual: "FloatingPoint",
}),
}
}
fn decode(self, bytes: &[u8], order: &DatatypeByteOrder) -> f64 {
match self {
FloatFormat::Half => f64::from(read_f16_bytes(bytes, order)),
FloatFormat::Single => f64::from(read_f32_bytes(bytes, order)),
FloatFormat::Double => read_f64_bytes(bytes, order),
FloatFormat::Other(layout) => {
layout.decode(read_unsigned_int(bytes, bytes.len(), order))
}
}
}
}
impl FloatLayout {
/// Decode the value held in the low `size * 8` bits of `bits`.
fn decode(self, bits: u64) -> f64 {
let field = |location: u32, size: u32| (bits >> location) & ((1u64 << size) - 1);
let exponent = field(self.exponent_location, self.exponent_size);
let mantissa = field(self.mantissa_location, self.mantissa_size);
let negative = field(self.exponent_location + self.exponent_size, 1) == 1;
let max_exponent = (1u64 << self.exponent_size) - 1;
let magnitude = if exponent == max_exponent {
if mantissa == 0 {
f64::INFINITY
} else {
f64::NAN
}
} else {
let bias = i64::from(self.exponent_bias);
let msize = i64::from(self.mantissa_size);
// value = significand * 2^power, with an implied leading 1 unless
// the number is subnormal (exponent field 0).
let (significand, power) = if exponent == 0 {
(mantissa, 1 - bias - msize)
} else {
(
mantissa | (1u64 << self.mantissa_size),
exponent as i64 - bias - msize,
)
};
scale_by_pow2(significand as f64, power)
};
if negative { -magnitude } else { magnitude }
}
}
/// `x * 2^power` without `std` (no `powi`/`libm`). `x` is a non-negative
/// integer below 2^53, so it is exact.
fn scale_by_pow2(x: f64, power: i64) -> f64 {
if x == 0.0 || power < -1200 {
return 0.0;
}
if power > 1100 {
return f64::INFINITY;
}
let pow2 = |p: i64| f64::from_bits(((p + 1023) as u64) << 52);
let mut x = x;
let mut power = power;
while power > 1023 {
x *= pow2(1023);
power -= 1023;
}
while power < -1022 {
x *= pow2(-1022);
power += 1022;
}
x * pow2(power)
}
/// Whether `datatype` is the little-endian IEEE float `format`, whose bytes
/// can be copied straight into native values on a little-endian target.
fn is_native_le_float(datatype: &Datatype, format: FloatFormat) -> bool {
matches!(
datatype,
Datatype::FloatingPoint {
byte_order: DatatypeByteOrder::LittleEndian,
..
}
) && FloatFormat::of(datatype).is_ok_and(|f| f == format)
}
fn read_f64_bytes(bytes: &[u8], order: &DatatypeByteOrder) -> f64 {
let buf = reorder_bytes(bytes, order);
f64::from_le_bytes(buf)
@@ -1666,20 +1883,6 @@ fn effective_bits(size: usize, bit_offset: u16, bit_precision: u16) -> (u32, u32
(bit_offset as u32, prec)
}
/// `(bit_offset, bit_precision)` for a fixed-point datatype, full width for
/// other types.
fn fixed_bits(datatype: &Datatype) -> (u32, u32) {
match datatype {
Datatype::FixedPoint {
size,
bit_offset,
bit_precision,
..
} => effective_bits(*size as usize, *bit_offset, *bit_precision),
_ => (0, 0),
}
}
/// Whether a datatype occupies its full storage width (bit offset 0, precision
/// == size·8), in which case the bulk-copy fast read paths apply. Non
/// fixed-point types are treated as full width.
@@ -1892,6 +2095,67 @@ mod tests {
assert_eq!(read_as_u64(&raw, &dt).unwrap(), vec![4095, 1, 2048]);
}
#[test]
fn float_to_int_truncates_and_saturates() {
// Values libhdf5 hands to an undefined C cast: NaN reads as 0 and
// exactly 2^63 saturates instead of wrapping to i64::MIN.
let dt = make_f64_le_type();
let vals = [f64::NAN, 2f64.powi(63), -2.5, 2.0f64.powi(64)];
let raw: Vec<u8> = vals.iter().flat_map(|v| v.to_le_bytes()).collect();
assert_eq!(
read_as_i64(&raw, &dt).unwrap(),
vec![0, i64::MAX, -2, i64::MAX]
);
assert_eq!(
read_as_u64(&raw, &dt).unwrap(),
vec![0, 1 << 63, 0, u64::MAX]
);
assert_eq!(
read_as_i32(&raw, &dt).unwrap(),
vec![0, i32::MAX, -2, i32::MAX]
);
}
#[test]
fn bfloat16_and_fp8_decode_by_fields() {
// bfloat16 is a 2-byte float that is not IEEE half.
let bf16 = Datatype::FloatingPoint {
size: 2,
byte_order: DatatypeByteOrder::LittleEndian,
bit_offset: 0,
bit_precision: 16,
exponent_location: 7,
exponent_size: 8,
mantissa_location: 0,
mantissa_size: 7,
exponent_bias: 127,
};
let raw: Vec<u8> = [0x3FC0u16, 0xC010, 0x7F80, 0x0001]
.iter()
.flat_map(|v| v.to_le_bytes())
.collect();
let got = read_as_f64(&raw, &bf16).unwrap();
assert_eq!(&got[..3], &[1.5, -2.25, f64::INFINITY]);
assert_eq!(got[3], 2f64.powi(-133)); // smallest subnormal
assert_eq!(read_as_f32(&raw, &bf16).unwrap()[..2], [1.5, -2.25]);
// FP8 E4M3: 1, -1, 2, 0, NaN (IEEE-style, as libhdf5 treats it).
let e4m3 = Datatype::FloatingPoint {
size: 1,
byte_order: DatatypeByteOrder::LittleEndian,
bit_offset: 0,
bit_precision: 8,
exponent_location: 3,
exponent_size: 4,
mantissa_location: 0,
mantissa_size: 3,
exponent_bias: 7,
};
let got = read_as_f64(&[0x38, 0xB8, 0x40, 0x00, 0x7E], &e4m3).unwrap();
assert_eq!(&got[..4], &[1.0, -1.0, 2.0, 0.0]);
assert!(got[4].is_nan());
}
#[test]
fn full_width_signed_unchanged() {
// Regression: full-width 32-bit signed must be unaffected.
crates/clawhdf5-format/src/vl_data.rs
+10 -3
View File
@@ -148,7 +148,12 @@ pub fn read_vl_strings(
Ok(result)
}
/// Resolve VL byte sequences from raw data.
/// Resolve VL sequences from raw data, returning each element's bytes.
///
/// Each element is the sequence's full encoding — element count × base type
/// size bytes, in the base type's byte order — so a sequence of `i32` yields
/// four bytes per value. Decode it with the base type (e.g.
/// [`crate::data_read::read_as_i64`]).
pub fn read_vl_bytes(
file_data: &[u8],
raw_data: &[u8],
@@ -177,8 +182,10 @@ pub fn read_vl_bytes(
},
)?;
let len = (vl.length as usize).min(obj.data.len());
result.push(obj.data[..len].to_vec());
// The heap object holds the whole sequence. `vl.length` counts
// elements, not bytes, so it is only the byte length when the base
// type is one byte wide.
result.push(obj.data.clone());
}
Ok(result)