quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix(format): scale-offset and shuffle decode chunks as libhdf5 does

Browse Source 
Scale-offset (filter 6) now follows H5Z__filter_scaleoffset:
- the packed codes start at byte 21 whatever size the chunk records for
  minval (libhdf5 reads min(8, size) bytes of minval and always starts the
  codes at buf_offset 21). We started them after minval plus 8 bytes, so a
  chunk recording a size of 0 (cve-2025-44905 /Scale_offset_short_data_be)
  decoded differently from h5py;
- with a fill value defined, a code equal to the all-ones code of minbits
  bits is the fill value, including minbits 0 (code 0): a chunk of nothing
  but fill values read as minval;
- minbits of the full width stores the elements as they are (no minval
  added), and an integer scale factor of the full width means the chunk was
  left untouched; minbits or a scale factor wider than the type is an
  error;
- the class parameter (integer or float) decides the decode, a scale type
  that does not match it is refused, and E-scale is refused, as in libhdf5
  (no library writes it; it was decoded here unchecked);
- minval is the stored bytes zero-extended, as libhdf5 reads it.
Codes past the end of the chunk stay an error, as in libhdf5 releases
after 2.0 ("Buffer too short"; 2.0 reads past the buffer, cve-2025-2308).

Shuffle (filter 2) uses its own parameter as the element size, as libhdf5
does, instead of the dataset's element size; a parameter larger than the
chunk leaves the chunk as it is (cve-2025-44905 /Shuffle_float_data_be),
and a parameter of 0 is an error.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-26 10:32:03 -05:00
co-authored by Claude Opus 5.5
parent b3058ca46e
commit d110b1d945
1 changed files with 236 additions and 140 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/filters.rs
+235 -139
View File
@@ -183,7 +183,7 @@ pub(crate) static BUILTIN_FILTERS: &[BuiltinFilter] = &[
BuiltinFilter {
id: FILTER_SHUFFLE,
name: "shuffle",
decode: |d, c| shuffle_decompress(d, c.element_size),
decode: |d, c| shuffle_decompress(d, shuffle_type_size(c.client_data(), c.element_size)?),
encode: Some(|d, c| shuffle_compress(d, c.element_size)),
},
BuiltinFilter {
@@ -280,23 +280,6 @@ pub(crate) static BUILTIN_FILTERS: &[BuiltinFilter] = &[
},
];
/// Decode the HDF5 scale-offset filter (id 6).
///
/// Supports all three scale-offset variants:
/// - `H5Z_SO_FLOAT_DSCALE` (0): `value = minval + code / 10^D`
/// - `H5Z_SO_FLOAT_ESCALE` (1): `value = minval + code * 2^E`
/// - `H5Z_SO_INT` (2): `value = minval + code`
///
/// Compressed buffer layout: `minbits` (u32 LE) · `minval_width` (1 byte)
/// · `minval` (`minval_width` bytes) · 8 reserved bytes · MSB-first packed
/// codes (`nelmts * minbits` bits). The all-ones code is reserved for the
/// defined fill value.
///
/// `cd` is the `H5Zscaleoffset.c` parameter block: `[0]`=scale type,
/// `[1]`=scale factor (decimal digits D for D-scale, binary exponent E for
/// E-scale, interpreted as i32 for negative exponents), `[2]`=element count,
/// `[4]`=element size, `[5]`=signed flag, `[6]`=byte order (1 = big-endian),
/// `[7]`=fill defined, `[8..]`=fill value bits.
/// `f64::powi` equivalent that works under `no_std` (no libm/std available).
/// Exponentiation by squaring, matching `powi`'s semantics for negative
/// exponents via reciprocal.
@@ -318,6 +301,32 @@ fn powi_f64(base: f64, mut exp: i32) -> f64 {
if neg { 1.0 / result } else { result }
}
/// Decode the HDF5 scale-offset filter (id 6) as libhdf5 does
/// (`H5Z__filter_scaleoffset`, reverse direction).
///
/// - Integers (`H5Z_SO_INT`): `value = minval + code`.
/// - Floats, D-scale (`H5Z_SO_FLOAT_DSCALE`): `value = code / 10^D + min`.
/// - Floats, E-scale: refused, as libhdf5 refuses it ("E-scaling method not
/// supported"); no library writes it.
///
/// Compressed buffer layout: `minbits` (u32 LE) · the size of `minval` in
/// bytes (1 byte; libhdf5 uses at most 8 of them) · `minval` · packed codes
/// at byte 21, whatever the stored size of `minval` (`buf_offset` is fixed)
/// · MSB-first, `minbits` bits per element. With a fill value defined, the
/// all-ones code of `minbits` bits is the fill value — for `minbits == 0`
/// that is every element. `minbits` equal to the element's full width means
/// the elements are stored as they are (in little-endian order), and an
/// integer scale factor of the full width means the filter left the chunk
/// untouched.
///
/// `cd` is the `H5Zscaleoffset.c` parameter block: `[0]` scale type, `[1]`
/// scale factor, `[2]` element count, `[3]` class (0 integer, 1 float),
/// `[4]` element size, `[5]` signed, `[6]` byte order (1 = big-endian), `[7]`
/// fill defined, `[8..]` fill value bits.
///
/// Packed data too short for its codes is an error (libhdf5 2.0 read past
/// the end of the chunk buffer — `cve-2025-2308` — and later releases
/// refuse it: "Buffer too short").
fn scaleoffset_decompress(
data: &[u8],
cd: &[u32],
@@ -326,74 +335,101 @@ fn scaleoffset_decompress(
const H5Z_SO_FLOAT_DSCALE: u32 = 0;
const H5Z_SO_FLOAT_ESCALE: u32 = 1;
const H5Z_SO_INT: u32 = 2;
/// Where the packed codes start (`buf_offset` in `H5Zscaleoffset.c`).
const BUF_OFFSET: usize = 21;
let err = |why: &str| FormatError::ChunkedReadError(format!("scale-offset: {why}"));
if cd.len() < 8 {
return Err(FormatError::ChunkedReadError(
"scale-offset: missing filter client data".into(),
));
return Err(err("missing filter client data"));
}
let scale_type = cd[0];
let is_float = scale_type == H5Z_SO_FLOAT_DSCALE || scale_type == H5Z_SO_FLOAT_ESCALE;
if scale_type != H5Z_SO_INT && !is_float {
return Err(FormatError::UnsupportedFilter(FILTER_SCALEOFFSET));
let is_float = match cd[3] {
0 => false,
1 => true,
_ => return Err(err("cannot use C integer datatype for cast")),
};
if is_float && scale_type != H5Z_SO_FLOAT_DSCALE && scale_type != H5Z_SO_FLOAT_ESCALE
|| !is_float && scale_type != H5Z_SO_INT
{
return Err(err("invalid scale type"));
}
if scale_type == H5Z_SO_FLOAT_ESCALE {
return Err(err("E-scaling method not supported"));
}
let nelmts = cd[2] as usize;
let elem_size = cd[4] as usize;
if elem_size == 0 || elem_size > 8 || (is_float && elem_size != 4 && elem_size != 8) {
return Err(FormatError::ChunkedReadError(
"scale-offset: unsupported element size".into(),
));
let size_ok = if is_float {
matches!(elem_size, 4 | 8)
} else {
matches!(elem_size, 1 | 2 | 4 | 8)
};
if !size_ok {
return Err(err("cannot use C integer datatype for cast"));
}
let full_bits = elem_size * 8;
// An integer's scale factor is the number of bits kept; all of them
// means the filter stored the chunk as it was.
if !is_float && (cd[1] as i32).max(0) as usize > full_bits {
return Err(err("minimum number of bits exceeds maximum"));
}
if !is_float && cd[1] as i32 == full_bits as i32 {
return Ok(data.to_vec());
}
// The decoded output must match the chunk's uncompressed size; reject an
// element count that would over-allocate (e.g. minbits == 0 with a huge
// nelmts and no packed payload to bound it).
let out_bytes = nelmts
.checked_mul(elem_size)
.ok_or_else(|| FormatError::ChunkedReadError("scale-offset: size overflow".into()))?;
.ok_or_else(|| err("size overflow"))?;
if expected_bytes != 0 && out_bytes > expected_bytes {
return Err(FormatError::ChunkedReadError(
"scale-offset: element count exceeds chunk size".into(),
));
return Err(err("element count exceeds chunk size"));
}
let signed = cd[5] == 1;
let big_endian = cd[6] == 1;
let fill_defined = cd[7] == 1;
// --- header: minbits, then minval, then 8 reserved bytes ---
// --- header: minbits, then the size of minval and minval ---
if data.len() < 5 {
return Err(FormatError::ChunkedReadError(
"scale-offset: truncated header".into(),
));
return Err(err("buffer too short"));
}
let minbits = u32::from_le_bytes([data[0], data[1], data[2], data[3]]) as usize;
let minval_width = data[4] as usize;
let minval_end = 5 + minval_width;
if data.len() < minval_end {
return Err(FormatError::ChunkedReadError(
"scale-offset: truncated minval".into(),
));
if minbits > full_bits {
return Err(err("minimum number of bits exceeds size of type"));
}
let minval_bytes = &data[5..minval_end];
let minval_size = usize::from(data[4]).min(8);
let minval_bytes = data
.get(5..5 + minval_size)
.ok_or_else(|| err("buffer too short"))?;
let minval = minval_bytes
.iter()
.rev()
.fold(0u64, |acc, &b| (acc << 8) | u64::from(b));
// --- unpack the per-element codes (MSB-first), shared by both variants ---
if minbits > 64 {
return Err(FormatError::ChunkedReadError(
"scale-offset: implausible minbits".into(),
));
// Full precision: the elements follow as they were, little-endian.
if minbits == full_bits {
let raw = data
.get(BUF_OFFSET..)
.and_then(|d| d.get(..out_bytes))
.ok_or_else(|| err("buffer too short"))?;
let mut out = raw.to_vec();
if big_endian {
for e in out.chunks_exact_mut(elem_size) {
e.reverse();
}
}
return Ok(out);
}
// --- unpack the per-element codes (MSB-first) ---
let codes: Vec<u64> = if minbits == 0 {
// No packed payload: every element equals minval.
// No packed payload: every code is 0.
vec![0u64; nelmts]
} else {
let packed = data.get(minval_end + 8..).ok_or_else(|| {
FormatError::ChunkedReadError("scale-offset: truncated packed data".into())
})?;
let packed = data.get(BUF_OFFSET..).unwrap_or(&[]);
let need_bits = nelmts
.checked_mul(minbits)
.ok_or_else(|| FormatError::ChunkedReadError("scale-offset: size overflow".into()))?;
.ok_or_else(|| err("size overflow"))?;
if packed.len() * 8 < need_bits {
return Err(FormatError::ChunkedReadError(
"scale-offset: packed data too short".into(),
));
return Err(err("packed data too short"));
}
let mut out = Vec::with_capacity(nelmts);
let mut bitpos = 0usize;
@@ -408,35 +444,28 @@ fn scaleoffset_decompress(
}
out
};
// The fill code (all ones) only exists when there are bits to pack.
let has_fill_code = fill_defined && minbits > 0 && minbits < 64;
// Computed for all 1..=64 widths; `1 << 64` would overflow, so saturate.
let fill_code: u64 = if minbits == 0 {
0
} else if minbits >= 64 {
u64::MAX
} else {
(1u64 << minbits) - 1
// With a fill value defined, the all-ones code of `minbits` bits (0
// when minbits is 0) stands for it. minbits < 64 here.
let fill_code: u64 = (1u64 << minbits) - 1;
let fill_bits = || {
let lo = u64::from(*cd.get(8).unwrap_or(&0));
let hi = u64::from(*cd.get(9).unwrap_or(&0));
lo | (hi << 32)
};
if is_float {
let is_escale = scale_type == H5Z_SO_FLOAT_ESCALE;
let scale_factor = cd[1] as i32;
let minval = read_le_float(minval_bytes, elem_size);
let minval = bits_to_float(minval, elem_size);
let fill_value = if fill_defined {
let lo = *cd.get(8).unwrap_or(&0) as u64;
let hi = *cd.get(9).unwrap_or(&0) as u64;
bits_to_float(lo | (hi << 32), elem_size)
bits_to_float(fill_bits(), elem_size)
} else {
0.0
};
let values: Vec<f64> = codes
.iter()
.map(|&code| {
if has_fill_code && code == fill_code {
if fill_defined && code == fill_code {
fill_value
} else if is_escale {
minval + code as f64 * powi_f64(2.0, scale_factor)
} else if elem_size == 4 {
// H5Z_scaleoffset_modify_3/4 for `float`: the code is
// read as an `int` and everything is single precision,
@@ -456,21 +485,19 @@ fn scaleoffset_decompress(
.collect();
Ok(write_floats(&values, elem_size, big_endian))
} else {
let minval = read_le_int(minval_bytes, signed);
let fill_value: i64 = if fill_defined {
let lo = *cd.get(8).unwrap_or(&0) as u64;
let hi = *cd.get(9).unwrap_or(&0) as u64;
sign_extend(lo | (hi << 32), elem_size, signed)
sign_extend(fill_bits(), elem_size, signed)
} else {
0
};
let values: Vec<i64> = codes
.iter()
.map(|&code| {
if has_fill_code && code == fill_code {
if fill_defined && code == fill_code {
fill_value
} else {
minval.wrapping_add(code as i64)
// `(type)(buf[i] + minval)`: wraps at the element width.
(code.wrapping_add(minval)) as i64
}
})
.collect();
@@ -478,21 +505,6 @@ fn scaleoffset_decompress(
}
}
/// Read a little-endian float of `size` bytes (4 = f32, otherwise f64) as f64.
fn read_le_float(bytes: &[u8], size: usize) -> f64 {
if size == 4 {
let mut b = [0u8; 4];
let n = bytes.len().min(4);
b[..n].copy_from_slice(&bytes[..n]);
f32::from_le_bytes(b) as f64
} else {
let mut b = [0u8; 8];
let n = bytes.len().min(8);
b[..n].copy_from_slice(&bytes[..n]);
f64::from_le_bytes(b)
}
}
/// Interpret the low bits of `raw` as an IEEE float of `size` bytes.
fn bits_to_float(raw: u64, size: usize) -> f64 {
if size == 4 {
@@ -525,16 +537,6 @@ fn write_floats(values: &[f64], elem_size: usize, big_endian: bool) -> Vec<u8> {
out
}
/// Read a little-endian integer of `bytes.len()` bytes, sign-extending when
/// `signed`. Used for the scale-offset `minval` field.
fn read_le_int(bytes: &[u8], signed: bool) -> i64 {
let mut raw: u64 = 0;
for (i, &b) in bytes.iter().enumerate().take(8) {
raw |= (b as u64) << (i * 8);
}
sign_extend(raw, bytes.len().min(8), signed)
}
/// Interpret the low `size` bytes of `raw` as a (possibly signed) integer.
fn sign_extend(raw: u64, size: usize, signed: bool) -> i64 {
if size == 0 || size >= 8 {
@@ -1216,6 +1218,23 @@ fn zstd_compress(data: &[u8], level: u32) -> Result<Vec<u8>, FormatError> {
/// Unshuffle (decompress direction): reconstruct interleaved element bytes.
/// On disk: all byte-0s of each element together, then all byte-1s, etc.
/// Output: elements in natural order.
/// The element size the shuffle filter works with: its parameter, as
/// libhdf5 uses it (`H5Z__filter_shuffle`), not the dataset's element size.
/// They are the same in every file a library wrote; a corrupt parameter
/// larger than the chunk makes libhdf5 leave the chunk as it is, and so
/// does [`shuffle_decompress`] (`cve-2025-44905`'s `Shuffle_float_data_be`).
/// A zero parameter is an error ("invalid shuffle parameters"); a pipeline
/// without the parameter (never written by libhdf5) uses the element size.
fn shuffle_type_size(cd: &[u32], element_size: usize) -> Result<usize, FormatError> {
match cd {
[] => Ok(element_size),
[0] | [_, _, ..] => Err(FormatError::FilterError(
"invalid shuffle parameters".into(),
)),
[size] => Ok(*size as usize),
}
}
fn shuffle_decompress(data: &[u8], element_size: usize) -> Result<Vec<u8>, FormatError> {
if element_size <= 1 {
return Ok(data.to_vec());
@@ -2434,48 +2453,125 @@ mod tests {
}
}
fn as_f64(bytes: &[u8]) -> Vec<f64> {
bytes
.as_chunks::<8>()
.0
.iter()
.map(|c| f64::from_le_bytes(*c))
.collect()
/// E-scale: libhdf5 refuses it on read and write ("E-scaling method not
/// supported"); it was decoded here, never checked against anything.
#[test]
fn scaleoffset_float_escale_is_refused_as_in_libhdf5() {
let cd = [1u32, 1, 4, 1, 8, 0, 0, 0];
let mut raw = vec![2, 0, 0, 0, 8];
raw.extend_from_slice(&[0; 16]);
raw.push(0x1B);
assert!(scaleoffset_decompress(&raw, &cd, 0).is_err());
}
/// A scale type that does not match the class is refused, as libhdf5
/// refuses it ("invalid scale type").
#[test]
fn scaleoffset_float_escale_e1() {
// f64 [0.0, 2.0, 4.0, 6.0], E=1 (×2^1=2), fill_defined=0.
// cd: scale_type=1, E=1, nelmts=4, elem_size=8.
let cd = [1u32, 1, 4, 0, 8, 0, 0, 0];
let raw: &[u8] = &[
2, 0, 0, 0, // minbits=2
8, // minval_width=8
0, 0, 0, 0, 0, 0, 0, 0, // minval=0.0f64
0, 0, 0, 0, 0, 0, 0, 0, // 8 reserved bytes
0x1B, // packed codes: 00 01 10 11 MSB-first
];
let got = as_f64(&scaleoffset_decompress(raw, &cd, 0).unwrap());
assert_eq!(got, vec![0.0, 2.0, 4.0, 6.0]);
fn scaleoffset_scale_type_must_match_the_class() {
let mut raw = vec![2, 0, 0, 0, 8];
raw.extend_from_slice(&[0; 16]);
raw.push(0x1B);
assert!(scaleoffset_decompress(&raw, &[0, 0, 4, 0, 4, 1, 0, 0], 0).is_err());
assert!(scaleoffset_decompress(&raw, &[2, 0, 4, 1, 4, 1, 0, 0], 0).is_err());
assert!(scaleoffset_decompress(&raw, &[2, 0, 4, 7, 4, 1, 0, 0], 0).is_err());
}
/// `cve-2025-44905` `/Scale_offset_short_data_be`, chunk (4, 0): the
/// stored size of `minval` is 0. libhdf5 reads a `minval` of 0 and the
/// packed codes from byte 21 regardless; we read them from byte 13
/// (5 + size + 8), so the values differed from h5py's.
#[test]
fn scaleoffset_float_escale_neg_exp() {
// f64 [0.0, 0.5, 1.0, 1.5], E=-1 (×2^-1=0.5), fill_defined=0.
// cd[1] = 0xFFFF_FFFF which casts to i32 = -1.
let cd = [1u32, 0xFFFF_FFFF, 4, 0, 8, 0, 0, 0];
let raw: &[u8] = &[
2, 0, 0, 0, // minbits=2
8, // minval_width=8
0, 0, 0, 0, 0, 0, 0, 0, // minval=0.0f64
0, 0, 0, 0, 0, 0, 0, 0, // 8 reserved bytes
0x1B, // packed codes: 00 01 10 11 MSB-first
];
let got = as_f64(&scaleoffset_decompress(raw, &cd, 0).unwrap());
let exp = [0.0f64, 0.5, 1.0, 1.5];
for (g, e) in got.iter().zip(exp.iter()) {
assert!((g - e).abs() < 1e-9, "got {g} expected {e}");
fn scaleoffset_codes_start_at_byte_21_whatever_the_minval_size() {
// big-endian i16, 12 elements, fill -2 (cd 65534), minbits 3.
let mut cd = vec![2u32, 0, 12, 0, 2, 1, 1, 1, 65534];
cd.resize(20, 0);
let raw = unhex("0300000000d20e00000000000034000000000000000400000000");
let got = scaleoffset_decompress(&raw, &cd, 24).unwrap();
// Codes of 3 bits from byte 21 (04 00 00 00 00): 0, 1, 0, ...; h5py
// reads the chunk's first row as 0, 1, 0.
let want: Vec<i16> = vec![0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0];
let want: Vec<u8> = want.iter().flat_map(|v| v.to_be_bytes()).collect();
assert_eq!(got, want);
}
/// With a fill value defined, libhdf5 compares each code with the
/// all-ones code of `minbits` bits — which for `minbits == 0` is 0, so a
/// chunk with no packed codes reads as all fill values (the compressor
/// writes that for a chunk of nothing but fill values). It read as
/// `minval` here.
#[test]
fn scaleoffset_minbits_zero_with_a_fill_value_is_all_fill() {
let mut cd = vec![2u32, 0, 3, 0, 4, 1, 0, 1, (-7i32) as u32];
cd.resize(20, 0);
let mut raw = vec![0, 0, 0, 0, 8];
raw.extend_from_slice(&5i64.to_le_bytes());
raw.extend_from_slice(&[0; 8]);
assert_eq!(
scaleoffset_decompress(&raw, &cd, 12).unwrap(),
i32_le(&[-7, -7, -7])
);
// Without a fill value every element is minval.
cd[7] = 0;
assert_eq!(
scaleoffset_decompress(&raw, &cd, 12).unwrap(),
i32_le(&[5, 5, 5])
);
}
/// `minbits` of the full width stores the elements as they are
/// (little-endian), without `minval`; a full-width integer scale factor
/// means the filter left the chunk untouched.
#[test]
fn scaleoffset_full_width_is_stored_as_is() {
let mut cd = vec![2u32, 0, 2, 0, 2, 1, 1, 0];
cd.resize(20, 0);
let mut raw = vec![16, 0, 0, 0, 8];
raw.extend_from_slice(&100i64.to_le_bytes());
raw.extend_from_slice(&[0; 8]);
raw.extend_from_slice(&[0x34, 0x12, 0xfe, 0xff]);
assert_eq!(
scaleoffset_decompress(&raw, &cd, 4).unwrap(),
[0x12, 0x34, 0xff, 0xfe]
);
cd[1] = 16;
assert_eq!(
scaleoffset_decompress(&[1, 2, 3, 4], &cd, 4).unwrap(),
[1, 2, 3, 4]
);
cd[1] = 17;
assert!(scaleoffset_decompress(&[1, 2, 3, 4], &cd, 4).is_err());
// minbits wider than the type.
cd[1] = 0;
raw[0] = 17;
assert!(scaleoffset_decompress(&raw, &cd, 4).is_err());
}
/// The shuffle filter uses its own parameter as the element size, as
/// libhdf5 does; a parameter larger than the chunk leaves the chunk as
/// it is (`cve-2025-44905` `/Shuffle_float_data_be`, whose parameter is
/// 4261347332: h5py and h5dump read the stored bytes unshuffled).
#[test]
fn shuffle_uses_its_parameter() {
let data: Vec<u8> = (0..16).collect();
let shuffled = shuffle_compress(&data, 4).unwrap();
let pipeline = |cd: Vec<u32>| FilterPipeline {
version: 2,
filters: vec![one_filter(FILTER_SHUFFLE, cd)],
};
// The dataset's element size says 2; the parameter says 4.
assert_eq!(
decompress_chunk(&shuffled, &pipeline(vec![4]), 16, 2).unwrap(),
data
);
assert_eq!(
decompress_chunk(&shuffled, &pipeline(vec![4_261_347_332]), 16, 4).unwrap(),
shuffled
);
assert!(decompress_chunk(&shuffled, &pipeline(vec![0]), 16, 4).is_err());
assert_eq!(
decompress_chunk(&shuffled, &pipeline(vec![]), 16, 4).unwrap(),
data
);
}
// --- N-Bit (filter id 5) --------------------------------------------------