From d110b1d9451945162356267e6f49b0b607ea8e63 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 10:31:52 -0500 Subject: [PATCH] fix(format): scale-offset and shuffle decode chunks as libhdf5 does Scale-offset (filter 6) now follows H5Z__filter_scaleoffset: - the packed codes start at byte 21 whatever size the chunk records for minval (libhdf5 reads min(8, size) bytes of minval and always starts the codes at buf_offset 21). We started them after minval plus 8 bytes, so a chunk recording a size of 0 (cve-2025-44905 /Scale_offset_short_data_be) decoded differently from h5py; - with a fill value defined, a code equal to the all-ones code of minbits bits is the fill value, including minbits 0 (code 0): a chunk of nothing but fill values read as minval; - minbits of the full width stores the elements as they are (no minval added), and an integer scale factor of the full width means the chunk was left untouched; minbits or a scale factor wider than the type is an error; - the class parameter (integer or float) decides the decode, a scale type that does not match it is refused, and E-scale is refused, as in libhdf5 (no library writes it; it was decoded here unchecked); - minval is the stored bytes zero-extended, as libhdf5 reads it. Codes past the end of the chunk stay an error, as in libhdf5 releases after 2.0 ("Buffer too short"; 2.0 reads past the buffer, cve-2025-2308). Shuffle (filter 2) uses its own parameter as the element size, as libhdf5 does, instead of the dataset's element size; a parameter larger than the chunk leaves the chunk as it is (cve-2025-44905 /Shuffle_float_data_be), and a parameter of 0 is an error. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/filters.rs | 376 ++++++++++++++++---------- 1 file changed, 236 insertions(+), 140 deletions(-) diff --git a/crates/clawhdf5-format/src/filters.rs b/crates/clawhdf5-format/src/filters.rs index 36f2e96..45b54b2 100644 --- a/crates/clawhdf5-format/src/filters.rs +++ b/crates/clawhdf5-format/src/filters.rs @@ -183,7 +183,7 @@ pub(crate) static BUILTIN_FILTERS: &[BuiltinFilter] = &[ BuiltinFilter { id: FILTER_SHUFFLE, name: "shuffle", - decode: |d, c| shuffle_decompress(d, c.element_size), + decode: |d, c| shuffle_decompress(d, shuffle_type_size(c.client_data(), c.element_size)?), encode: Some(|d, c| shuffle_compress(d, c.element_size)), }, BuiltinFilter { @@ -280,23 +280,6 @@ pub(crate) static BUILTIN_FILTERS: &[BuiltinFilter] = &[ }, ]; -/// Decode the HDF5 scale-offset filter (id 6). -/// -/// Supports all three scale-offset variants: -/// - `H5Z_SO_FLOAT_DSCALE` (0): `value = minval + code / 10^D` -/// - `H5Z_SO_FLOAT_ESCALE` (1): `value = minval + code * 2^E` -/// - `H5Z_SO_INT` (2): `value = minval + code` -/// -/// Compressed buffer layout: `minbits` (u32 LE) · `minval_width` (1 byte) -/// · `minval` (`minval_width` bytes) · 8 reserved bytes · MSB-first packed -/// codes (`nelmts * minbits` bits). The all-ones code is reserved for the -/// defined fill value. -/// -/// `cd` is the `H5Zscaleoffset.c` parameter block: `[0]`=scale type, -/// `[1]`=scale factor (decimal digits D for D-scale, binary exponent E for -/// E-scale, interpreted as i32 for negative exponents), `[2]`=element count, -/// `[4]`=element size, `[5]`=signed flag, `[6]`=byte order (1 = big-endian), -/// `[7]`=fill defined, `[8..]`=fill value bits. /// `f64::powi` equivalent that works under `no_std` (no libm/std available). /// Exponentiation by squaring, matching `powi`'s semantics for negative /// exponents via reciprocal. @@ -318,6 +301,32 @@ fn powi_f64(base: f64, mut exp: i32) -> f64 { if neg { 1.0 / result } else { result } } +/// Decode the HDF5 scale-offset filter (id 6) as libhdf5 does +/// (`H5Z__filter_scaleoffset`, reverse direction). +/// +/// - Integers (`H5Z_SO_INT`): `value = minval + code`. +/// - Floats, D-scale (`H5Z_SO_FLOAT_DSCALE`): `value = code / 10^D + min`. +/// - Floats, E-scale: refused, as libhdf5 refuses it ("E-scaling method not +/// supported"); no library writes it. +/// +/// Compressed buffer layout: `minbits` (u32 LE) · the size of `minval` in +/// bytes (1 byte; libhdf5 uses at most 8 of them) · `minval` · packed codes +/// at byte 21, whatever the stored size of `minval` (`buf_offset` is fixed) +/// · MSB-first, `minbits` bits per element. With a fill value defined, the +/// all-ones code of `minbits` bits is the fill value — for `minbits == 0` +/// that is every element. `minbits` equal to the element's full width means +/// the elements are stored as they are (in little-endian order), and an +/// integer scale factor of the full width means the filter left the chunk +/// untouched. +/// +/// `cd` is the `H5Zscaleoffset.c` parameter block: `[0]` scale type, `[1]` +/// scale factor, `[2]` element count, `[3]` class (0 integer, 1 float), +/// `[4]` element size, `[5]` signed, `[6]` byte order (1 = big-endian), `[7]` +/// fill defined, `[8..]` fill value bits. +/// +/// Packed data too short for its codes is an error (libhdf5 2.0 read past +/// the end of the chunk buffer — `cve-2025-2308` — and later releases +/// refuse it: "Buffer too short"). fn scaleoffset_decompress( data: &[u8], cd: &[u32], @@ -326,74 +335,101 @@ fn scaleoffset_decompress( const H5Z_SO_FLOAT_DSCALE: u32 = 0; const H5Z_SO_FLOAT_ESCALE: u32 = 1; const H5Z_SO_INT: u32 = 2; + /// Where the packed codes start (`buf_offset` in `H5Zscaleoffset.c`). + const BUF_OFFSET: usize = 21; + let err = |why: &str| FormatError::ChunkedReadError(format!("scale-offset: {why}")); if cd.len() < 8 { - return Err(FormatError::ChunkedReadError( - "scale-offset: missing filter client data".into(), - )); + return Err(err("missing filter client data")); } let scale_type = cd[0]; - let is_float = scale_type == H5Z_SO_FLOAT_DSCALE || scale_type == H5Z_SO_FLOAT_ESCALE; - if scale_type != H5Z_SO_INT && !is_float { - return Err(FormatError::UnsupportedFilter(FILTER_SCALEOFFSET)); + let is_float = match cd[3] { + 0 => false, + 1 => true, + _ => return Err(err("cannot use C integer datatype for cast")), + }; + if is_float && scale_type != H5Z_SO_FLOAT_DSCALE && scale_type != H5Z_SO_FLOAT_ESCALE + || !is_float && scale_type != H5Z_SO_INT + { + return Err(err("invalid scale type")); + } + if scale_type == H5Z_SO_FLOAT_ESCALE { + return Err(err("E-scaling method not supported")); } let nelmts = cd[2] as usize; let elem_size = cd[4] as usize; - if elem_size == 0 || elem_size > 8 || (is_float && elem_size != 4 && elem_size != 8) { - return Err(FormatError::ChunkedReadError( - "scale-offset: unsupported element size".into(), - )); + let size_ok = if is_float { + matches!(elem_size, 4 | 8) + } else { + matches!(elem_size, 1 | 2 | 4 | 8) + }; + if !size_ok { + return Err(err("cannot use C integer datatype for cast")); + } + let full_bits = elem_size * 8; + // An integer's scale factor is the number of bits kept; all of them + // means the filter stored the chunk as it was. + if !is_float && (cd[1] as i32).max(0) as usize > full_bits { + return Err(err("minimum number of bits exceeds maximum")); + } + if !is_float && cd[1] as i32 == full_bits as i32 { + return Ok(data.to_vec()); } // The decoded output must match the chunk's uncompressed size; reject an // element count that would over-allocate (e.g. minbits == 0 with a huge // nelmts and no packed payload to bound it). let out_bytes = nelmts .checked_mul(elem_size) - .ok_or_else(|| FormatError::ChunkedReadError("scale-offset: size overflow".into()))?; + .ok_or_else(|| err("size overflow"))?; if expected_bytes != 0 && out_bytes > expected_bytes { - return Err(FormatError::ChunkedReadError( - "scale-offset: element count exceeds chunk size".into(), - )); + return Err(err("element count exceeds chunk size")); } let signed = cd[5] == 1; let big_endian = cd[6] == 1; let fill_defined = cd[7] == 1; - // --- header: minbits, then minval, then 8 reserved bytes --- + // --- header: minbits, then the size of minval and minval --- if data.len() < 5 { - return Err(FormatError::ChunkedReadError( - "scale-offset: truncated header".into(), - )); + return Err(err("buffer too short")); } let minbits = u32::from_le_bytes([data[0], data[1], data[2], data[3]]) as usize; - let minval_width = data[4] as usize; - let minval_end = 5 + minval_width; - if data.len() < minval_end { - return Err(FormatError::ChunkedReadError( - "scale-offset: truncated minval".into(), - )); + if minbits > full_bits { + return Err(err("minimum number of bits exceeds size of type")); } - let minval_bytes = &data[5..minval_end]; + let minval_size = usize::from(data[4]).min(8); + let minval_bytes = data + .get(5..5 + minval_size) + .ok_or_else(|| err("buffer too short"))?; + let minval = minval_bytes + .iter() + .rev() + .fold(0u64, |acc, &b| (acc << 8) | u64::from(b)); - // --- unpack the per-element codes (MSB-first), shared by both variants --- - if minbits > 64 { - return Err(FormatError::ChunkedReadError( - "scale-offset: implausible minbits".into(), - )); + // Full precision: the elements follow as they were, little-endian. + if minbits == full_bits { + let raw = data + .get(BUF_OFFSET..) + .and_then(|d| d.get(..out_bytes)) + .ok_or_else(|| err("buffer too short"))?; + let mut out = raw.to_vec(); + if big_endian { + for e in out.chunks_exact_mut(elem_size) { + e.reverse(); + } + } + return Ok(out); } + + // --- unpack the per-element codes (MSB-first) --- let codes: Vec = if minbits == 0 { - // No packed payload: every element equals minval. + // No packed payload: every code is 0. vec![0u64; nelmts] } else { - let packed = data.get(minval_end + 8..).ok_or_else(|| { - FormatError::ChunkedReadError("scale-offset: truncated packed data".into()) - })?; + let packed = data.get(BUF_OFFSET..).unwrap_or(&[]); let need_bits = nelmts .checked_mul(minbits) - .ok_or_else(|| FormatError::ChunkedReadError("scale-offset: size overflow".into()))?; + .ok_or_else(|| err("size overflow"))?; if packed.len() * 8 < need_bits { - return Err(FormatError::ChunkedReadError( - "scale-offset: packed data too short".into(), - )); + return Err(err("packed data too short")); } let mut out = Vec::with_capacity(nelmts); let mut bitpos = 0usize; @@ -408,35 +444,28 @@ fn scaleoffset_decompress( } out }; - // The fill code (all ones) only exists when there are bits to pack. - let has_fill_code = fill_defined && minbits > 0 && minbits < 64; - // Computed for all 1..=64 widths; `1 << 64` would overflow, so saturate. - let fill_code: u64 = if minbits == 0 { - 0 - } else if minbits >= 64 { - u64::MAX - } else { - (1u64 << minbits) - 1 + // With a fill value defined, the all-ones code of `minbits` bits (0 + // when minbits is 0) stands for it. minbits < 64 here. + let fill_code: u64 = (1u64 << minbits) - 1; + let fill_bits = || { + let lo = u64::from(*cd.get(8).unwrap_or(&0)); + let hi = u64::from(*cd.get(9).unwrap_or(&0)); + lo | (hi << 32) }; if is_float { - let is_escale = scale_type == H5Z_SO_FLOAT_ESCALE; let scale_factor = cd[1] as i32; - let minval = read_le_float(minval_bytes, elem_size); + let minval = bits_to_float(minval, elem_size); let fill_value = if fill_defined { - let lo = *cd.get(8).unwrap_or(&0) as u64; - let hi = *cd.get(9).unwrap_or(&0) as u64; - bits_to_float(lo | (hi << 32), elem_size) + bits_to_float(fill_bits(), elem_size) } else { 0.0 }; let values: Vec = codes .iter() .map(|&code| { - if has_fill_code && code == fill_code { + if fill_defined && code == fill_code { fill_value - } else if is_escale { - minval + code as f64 * powi_f64(2.0, scale_factor) } else if elem_size == 4 { // H5Z_scaleoffset_modify_3/4 for `float`: the code is // read as an `int` and everything is single precision, @@ -456,21 +485,19 @@ fn scaleoffset_decompress( .collect(); Ok(write_floats(&values, elem_size, big_endian)) } else { - let minval = read_le_int(minval_bytes, signed); let fill_value: i64 = if fill_defined { - let lo = *cd.get(8).unwrap_or(&0) as u64; - let hi = *cd.get(9).unwrap_or(&0) as u64; - sign_extend(lo | (hi << 32), elem_size, signed) + sign_extend(fill_bits(), elem_size, signed) } else { 0 }; let values: Vec = codes .iter() .map(|&code| { - if has_fill_code && code == fill_code { + if fill_defined && code == fill_code { fill_value } else { - minval.wrapping_add(code as i64) + // `(type)(buf[i] + minval)`: wraps at the element width. + (code.wrapping_add(minval)) as i64 } }) .collect(); @@ -478,21 +505,6 @@ fn scaleoffset_decompress( } } -/// Read a little-endian float of `size` bytes (4 = f32, otherwise f64) as f64. -fn read_le_float(bytes: &[u8], size: usize) -> f64 { - if size == 4 { - let mut b = [0u8; 4]; - let n = bytes.len().min(4); - b[..n].copy_from_slice(&bytes[..n]); - f32::from_le_bytes(b) as f64 - } else { - let mut b = [0u8; 8]; - let n = bytes.len().min(8); - b[..n].copy_from_slice(&bytes[..n]); - f64::from_le_bytes(b) - } -} - /// Interpret the low bits of `raw` as an IEEE float of `size` bytes. fn bits_to_float(raw: u64, size: usize) -> f64 { if size == 4 { @@ -525,16 +537,6 @@ fn write_floats(values: &[f64], elem_size: usize, big_endian: bool) -> Vec { out } -/// Read a little-endian integer of `bytes.len()` bytes, sign-extending when -/// `signed`. Used for the scale-offset `minval` field. -fn read_le_int(bytes: &[u8], signed: bool) -> i64 { - let mut raw: u64 = 0; - for (i, &b) in bytes.iter().enumerate().take(8) { - raw |= (b as u64) << (i * 8); - } - sign_extend(raw, bytes.len().min(8), signed) -} - /// Interpret the low `size` bytes of `raw` as a (possibly signed) integer. fn sign_extend(raw: u64, size: usize, signed: bool) -> i64 { if size == 0 || size >= 8 { @@ -1216,6 +1218,23 @@ fn zstd_compress(data: &[u8], level: u32) -> Result, FormatError> { /// Unshuffle (decompress direction): reconstruct interleaved element bytes. /// On disk: all byte-0s of each element together, then all byte-1s, etc. /// Output: elements in natural order. +/// The element size the shuffle filter works with: its parameter, as +/// libhdf5 uses it (`H5Z__filter_shuffle`), not the dataset's element size. +/// They are the same in every file a library wrote; a corrupt parameter +/// larger than the chunk makes libhdf5 leave the chunk as it is, and so +/// does [`shuffle_decompress`] (`cve-2025-44905`'s `Shuffle_float_data_be`). +/// A zero parameter is an error ("invalid shuffle parameters"); a pipeline +/// without the parameter (never written by libhdf5) uses the element size. +fn shuffle_type_size(cd: &[u32], element_size: usize) -> Result { + match cd { + [] => Ok(element_size), + [0] | [_, _, ..] => Err(FormatError::FilterError( + "invalid shuffle parameters".into(), + )), + [size] => Ok(*size as usize), + } +} + fn shuffle_decompress(data: &[u8], element_size: usize) -> Result, FormatError> { if element_size <= 1 { return Ok(data.to_vec()); @@ -2434,48 +2453,125 @@ mod tests { } } - fn as_f64(bytes: &[u8]) -> Vec { - bytes - .as_chunks::<8>() - .0 - .iter() - .map(|c| f64::from_le_bytes(*c)) - .collect() + /// E-scale: libhdf5 refuses it on read and write ("E-scaling method not + /// supported"); it was decoded here, never checked against anything. + #[test] + fn scaleoffset_float_escale_is_refused_as_in_libhdf5() { + let cd = [1u32, 1, 4, 1, 8, 0, 0, 0]; + let mut raw = vec![2, 0, 0, 0, 8]; + raw.extend_from_slice(&[0; 16]); + raw.push(0x1B); + assert!(scaleoffset_decompress(&raw, &cd, 0).is_err()); } + /// A scale type that does not match the class is refused, as libhdf5 + /// refuses it ("invalid scale type"). #[test] - fn scaleoffset_float_escale_e1() { - // f64 [0.0, 2.0, 4.0, 6.0], E=1 (×2^1=2), fill_defined=0. - // cd: scale_type=1, E=1, nelmts=4, elem_size=8. - let cd = [1u32, 1, 4, 0, 8, 0, 0, 0]; - let raw: &[u8] = &[ - 2, 0, 0, 0, // minbits=2 - 8, // minval_width=8 - 0, 0, 0, 0, 0, 0, 0, 0, // minval=0.0f64 - 0, 0, 0, 0, 0, 0, 0, 0, // 8 reserved bytes - 0x1B, // packed codes: 00 01 10 11 MSB-first - ]; - let got = as_f64(&scaleoffset_decompress(raw, &cd, 0).unwrap()); - assert_eq!(got, vec![0.0, 2.0, 4.0, 6.0]); + fn scaleoffset_scale_type_must_match_the_class() { + let mut raw = vec![2, 0, 0, 0, 8]; + raw.extend_from_slice(&[0; 16]); + raw.push(0x1B); + assert!(scaleoffset_decompress(&raw, &[0, 0, 4, 0, 4, 1, 0, 0], 0).is_err()); + assert!(scaleoffset_decompress(&raw, &[2, 0, 4, 1, 4, 1, 0, 0], 0).is_err()); + assert!(scaleoffset_decompress(&raw, &[2, 0, 4, 7, 4, 1, 0, 0], 0).is_err()); } + /// `cve-2025-44905` `/Scale_offset_short_data_be`, chunk (4, 0): the + /// stored size of `minval` is 0. libhdf5 reads a `minval` of 0 and the + /// packed codes from byte 21 regardless; we read them from byte 13 + /// (5 + size + 8), so the values differed from h5py's. #[test] - fn scaleoffset_float_escale_neg_exp() { - // f64 [0.0, 0.5, 1.0, 1.5], E=-1 (×2^-1=0.5), fill_defined=0. - // cd[1] = 0xFFFF_FFFF which casts to i32 = -1. - let cd = [1u32, 0xFFFF_FFFF, 4, 0, 8, 0, 0, 0]; - let raw: &[u8] = &[ - 2, 0, 0, 0, // minbits=2 - 8, // minval_width=8 - 0, 0, 0, 0, 0, 0, 0, 0, // minval=0.0f64 - 0, 0, 0, 0, 0, 0, 0, 0, // 8 reserved bytes - 0x1B, // packed codes: 00 01 10 11 MSB-first - ]; - let got = as_f64(&scaleoffset_decompress(raw, &cd, 0).unwrap()); - let exp = [0.0f64, 0.5, 1.0, 1.5]; - for (g, e) in got.iter().zip(exp.iter()) { - assert!((g - e).abs() < 1e-9, "got {g} expected {e}"); - } + fn scaleoffset_codes_start_at_byte_21_whatever_the_minval_size() { + // big-endian i16, 12 elements, fill -2 (cd 65534), minbits 3. + let mut cd = vec![2u32, 0, 12, 0, 2, 1, 1, 1, 65534]; + cd.resize(20, 0); + let raw = unhex("0300000000d20e00000000000034000000000000000400000000"); + let got = scaleoffset_decompress(&raw, &cd, 24).unwrap(); + // Codes of 3 bits from byte 21 (04 00 00 00 00): 0, 1, 0, ...; h5py + // reads the chunk's first row as 0, 1, 0. + let want: Vec = vec![0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + let want: Vec = want.iter().flat_map(|v| v.to_be_bytes()).collect(); + assert_eq!(got, want); + } + + /// With a fill value defined, libhdf5 compares each code with the + /// all-ones code of `minbits` bits — which for `minbits == 0` is 0, so a + /// chunk with no packed codes reads as all fill values (the compressor + /// writes that for a chunk of nothing but fill values). It read as + /// `minval` here. + #[test] + fn scaleoffset_minbits_zero_with_a_fill_value_is_all_fill() { + let mut cd = vec![2u32, 0, 3, 0, 4, 1, 0, 1, (-7i32) as u32]; + cd.resize(20, 0); + let mut raw = vec![0, 0, 0, 0, 8]; + raw.extend_from_slice(&5i64.to_le_bytes()); + raw.extend_from_slice(&[0; 8]); + assert_eq!( + scaleoffset_decompress(&raw, &cd, 12).unwrap(), + i32_le(&[-7, -7, -7]) + ); + // Without a fill value every element is minval. + cd[7] = 0; + assert_eq!( + scaleoffset_decompress(&raw, &cd, 12).unwrap(), + i32_le(&[5, 5, 5]) + ); + } + + /// `minbits` of the full width stores the elements as they are + /// (little-endian), without `minval`; a full-width integer scale factor + /// means the filter left the chunk untouched. + #[test] + fn scaleoffset_full_width_is_stored_as_is() { + let mut cd = vec![2u32, 0, 2, 0, 2, 1, 1, 0]; + cd.resize(20, 0); + let mut raw = vec![16, 0, 0, 0, 8]; + raw.extend_from_slice(&100i64.to_le_bytes()); + raw.extend_from_slice(&[0; 8]); + raw.extend_from_slice(&[0x34, 0x12, 0xfe, 0xff]); + assert_eq!( + scaleoffset_decompress(&raw, &cd, 4).unwrap(), + [0x12, 0x34, 0xff, 0xfe] + ); + cd[1] = 16; + assert_eq!( + scaleoffset_decompress(&[1, 2, 3, 4], &cd, 4).unwrap(), + [1, 2, 3, 4] + ); + cd[1] = 17; + assert!(scaleoffset_decompress(&[1, 2, 3, 4], &cd, 4).is_err()); + // minbits wider than the type. + cd[1] = 0; + raw[0] = 17; + assert!(scaleoffset_decompress(&raw, &cd, 4).is_err()); + } + + /// The shuffle filter uses its own parameter as the element size, as + /// libhdf5 does; a parameter larger than the chunk leaves the chunk as + /// it is (`cve-2025-44905` `/Shuffle_float_data_be`, whose parameter is + /// 4261347332: h5py and h5dump read the stored bytes unshuffled). + #[test] + fn shuffle_uses_its_parameter() { + let data: Vec = (0..16).collect(); + let shuffled = shuffle_compress(&data, 4).unwrap(); + let pipeline = |cd: Vec| FilterPipeline { + version: 2, + filters: vec![one_filter(FILTER_SHUFFLE, cd)], + }; + // The dataset's element size says 2; the parameter says 4. + assert_eq!( + decompress_chunk(&shuffled, &pipeline(vec![4]), 16, 2).unwrap(), + data + ); + assert_eq!( + decompress_chunk(&shuffled, &pipeline(vec![4_261_347_332]), 16, 4).unwrap(), + shuffled + ); + assert!(decompress_chunk(&shuffled, &pipeline(vec![0]), 16, 4).is_err()); + assert_eq!( + decompress_chunk(&shuffled, &pipeline(vec![]), 16, 4).unwrap(), + data + ); } // --- N-Bit (filter id 5) --------------------------------------------------