fix(format): scale-offset and shuffle decode chunks as libhdf5 does
Scale-offset (filter 6) now follows H5Z__filter_scaleoffset:
- the packed codes start at byte 21 whatever size the chunk records for
minval (libhdf5 reads min(8, size) bytes of minval and always starts the
codes at buf_offset 21). We started them after minval plus 8 bytes, so a
chunk recording a size of 0 (cve-2025-44905 /Scale_offset_short_data_be)
decoded differently from h5py;
- with a fill value defined, a code equal to the all-ones code of minbits
bits is the fill value, including minbits 0 (code 0): a chunk of nothing
but fill values read as minval;
- minbits of the full width stores the elements as they are (no minval
added), and an integer scale factor of the full width means the chunk was
left untouched; minbits or a scale factor wider than the type is an
error;
- the class parameter (integer or float) decides the decode, a scale type
that does not match it is refused, and E-scale is refused, as in libhdf5
(no library writes it; it was decoded here unchecked);
- minval is the stored bytes zero-extended, as libhdf5 reads it.
Codes past the end of the chunk stay an error, as in libhdf5 releases
after 2.0 ("Buffer too short"; 2.0 reads past the buffer, cve-2025-2308).
Shuffle (filter 2) uses its own parameter as the element size, as libhdf5
does, instead of the dataset's element size; a parameter larger than the
chunk leaves the chunk as it is (cve-2025-44905 /Shuffle_float_data_be),
and a parameter of 0 is an error.
Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
@@ -183,7 +183,7 @@ pub(crate) static BUILTIN_FILTERS: &[BuiltinFilter] = &[
|
|||||||
BuiltinFilter {
|
BuiltinFilter {
|
||||||
id: FILTER_SHUFFLE,
|
id: FILTER_SHUFFLE,
|
||||||
name: "shuffle",
|
name: "shuffle",
|
||||||
decode: |d, c| shuffle_decompress(d, c.element_size),
|
decode: |d, c| shuffle_decompress(d, shuffle_type_size(c.client_data(), c.element_size)?),
|
||||||
encode: Some(|d, c| shuffle_compress(d, c.element_size)),
|
encode: Some(|d, c| shuffle_compress(d, c.element_size)),
|
||||||
},
|
},
|
||||||
BuiltinFilter {
|
BuiltinFilter {
|
||||||
@@ -280,23 +280,6 @@ pub(crate) static BUILTIN_FILTERS: &[BuiltinFilter] = &[
|
|||||||
},
|
},
|
||||||
];
|
];
|
||||||
|
|
||||||
/// Decode the HDF5 scale-offset filter (id 6).
|
|
||||||
///
|
|
||||||
/// Supports all three scale-offset variants:
|
|
||||||
/// - `H5Z_SO_FLOAT_DSCALE` (0): `value = minval + code / 10^D`
|
|
||||||
/// - `H5Z_SO_FLOAT_ESCALE` (1): `value = minval + code * 2^E`
|
|
||||||
/// - `H5Z_SO_INT` (2): `value = minval + code`
|
|
||||||
///
|
|
||||||
/// Compressed buffer layout: `minbits` (u32 LE) · `minval_width` (1 byte)
|
|
||||||
/// · `minval` (`minval_width` bytes) · 8 reserved bytes · MSB-first packed
|
|
||||||
/// codes (`nelmts * minbits` bits). The all-ones code is reserved for the
|
|
||||||
/// defined fill value.
|
|
||||||
///
|
|
||||||
/// `cd` is the `H5Zscaleoffset.c` parameter block: `[0]`=scale type,
|
|
||||||
/// `[1]`=scale factor (decimal digits D for D-scale, binary exponent E for
|
|
||||||
/// E-scale, interpreted as i32 for negative exponents), `[2]`=element count,
|
|
||||||
/// `[4]`=element size, `[5]`=signed flag, `[6]`=byte order (1 = big-endian),
|
|
||||||
/// `[7]`=fill defined, `[8..]`=fill value bits.
|
|
||||||
/// `f64::powi` equivalent that works under `no_std` (no libm/std available).
|
/// `f64::powi` equivalent that works under `no_std` (no libm/std available).
|
||||||
/// Exponentiation by squaring, matching `powi`'s semantics for negative
|
/// Exponentiation by squaring, matching `powi`'s semantics for negative
|
||||||
/// exponents via reciprocal.
|
/// exponents via reciprocal.
|
||||||
@@ -318,6 +301,32 @@ fn powi_f64(base: f64, mut exp: i32) -> f64 {
|
|||||||
if neg { 1.0 / result } else { result }
|
if neg { 1.0 / result } else { result }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Decode the HDF5 scale-offset filter (id 6) as libhdf5 does
|
||||||
|
/// (`H5Z__filter_scaleoffset`, reverse direction).
|
||||||
|
///
|
||||||
|
/// - Integers (`H5Z_SO_INT`): `value = minval + code`.
|
||||||
|
/// - Floats, D-scale (`H5Z_SO_FLOAT_DSCALE`): `value = code / 10^D + min`.
|
||||||
|
/// - Floats, E-scale: refused, as libhdf5 refuses it ("E-scaling method not
|
||||||
|
/// supported"); no library writes it.
|
||||||
|
///
|
||||||
|
/// Compressed buffer layout: `minbits` (u32 LE) · the size of `minval` in
|
||||||
|
/// bytes (1 byte; libhdf5 uses at most 8 of them) · `minval` · packed codes
|
||||||
|
/// at byte 21, whatever the stored size of `minval` (`buf_offset` is fixed)
|
||||||
|
/// · MSB-first, `minbits` bits per element. With a fill value defined, the
|
||||||
|
/// all-ones code of `minbits` bits is the fill value — for `minbits == 0`
|
||||||
|
/// that is every element. `minbits` equal to the element's full width means
|
||||||
|
/// the elements are stored as they are (in little-endian order), and an
|
||||||
|
/// integer scale factor of the full width means the filter left the chunk
|
||||||
|
/// untouched.
|
||||||
|
///
|
||||||
|
/// `cd` is the `H5Zscaleoffset.c` parameter block: `[0]` scale type, `[1]`
|
||||||
|
/// scale factor, `[2]` element count, `[3]` class (0 integer, 1 float),
|
||||||
|
/// `[4]` element size, `[5]` signed, `[6]` byte order (1 = big-endian), `[7]`
|
||||||
|
/// fill defined, `[8..]` fill value bits.
|
||||||
|
///
|
||||||
|
/// Packed data too short for its codes is an error (libhdf5 2.0 read past
|
||||||
|
/// the end of the chunk buffer — `cve-2025-2308` — and later releases
|
||||||
|
/// refuse it: "Buffer too short").
|
||||||
fn scaleoffset_decompress(
|
fn scaleoffset_decompress(
|
||||||
data: &[u8],
|
data: &[u8],
|
||||||
cd: &[u32],
|
cd: &[u32],
|
||||||
@@ -326,74 +335,101 @@ fn scaleoffset_decompress(
|
|||||||
const H5Z_SO_FLOAT_DSCALE: u32 = 0;
|
const H5Z_SO_FLOAT_DSCALE: u32 = 0;
|
||||||
const H5Z_SO_FLOAT_ESCALE: u32 = 1;
|
const H5Z_SO_FLOAT_ESCALE: u32 = 1;
|
||||||
const H5Z_SO_INT: u32 = 2;
|
const H5Z_SO_INT: u32 = 2;
|
||||||
|
/// Where the packed codes start (`buf_offset` in `H5Zscaleoffset.c`).
|
||||||
|
const BUF_OFFSET: usize = 21;
|
||||||
|
let err = |why: &str| FormatError::ChunkedReadError(format!("scale-offset: {why}"));
|
||||||
if cd.len() < 8 {
|
if cd.len() < 8 {
|
||||||
return Err(FormatError::ChunkedReadError(
|
return Err(err("missing filter client data"));
|
||||||
"scale-offset: missing filter client data".into(),
|
|
||||||
));
|
|
||||||
}
|
}
|
||||||
let scale_type = cd[0];
|
let scale_type = cd[0];
|
||||||
let is_float = scale_type == H5Z_SO_FLOAT_DSCALE || scale_type == H5Z_SO_FLOAT_ESCALE;
|
let is_float = match cd[3] {
|
||||||
if scale_type != H5Z_SO_INT && !is_float {
|
0 => false,
|
||||||
return Err(FormatError::UnsupportedFilter(FILTER_SCALEOFFSET));
|
1 => true,
|
||||||
|
_ => return Err(err("cannot use C integer datatype for cast")),
|
||||||
|
};
|
||||||
|
if is_float && scale_type != H5Z_SO_FLOAT_DSCALE && scale_type != H5Z_SO_FLOAT_ESCALE
|
||||||
|
|| !is_float && scale_type != H5Z_SO_INT
|
||||||
|
{
|
||||||
|
return Err(err("invalid scale type"));
|
||||||
|
}
|
||||||
|
if scale_type == H5Z_SO_FLOAT_ESCALE {
|
||||||
|
return Err(err("E-scaling method not supported"));
|
||||||
}
|
}
|
||||||
let nelmts = cd[2] as usize;
|
let nelmts = cd[2] as usize;
|
||||||
let elem_size = cd[4] as usize;
|
let elem_size = cd[4] as usize;
|
||||||
if elem_size == 0 || elem_size > 8 || (is_float && elem_size != 4 && elem_size != 8) {
|
let size_ok = if is_float {
|
||||||
return Err(FormatError::ChunkedReadError(
|
matches!(elem_size, 4 | 8)
|
||||||
"scale-offset: unsupported element size".into(),
|
} else {
|
||||||
));
|
matches!(elem_size, 1 | 2 | 4 | 8)
|
||||||
|
};
|
||||||
|
if !size_ok {
|
||||||
|
return Err(err("cannot use C integer datatype for cast"));
|
||||||
|
}
|
||||||
|
let full_bits = elem_size * 8;
|
||||||
|
// An integer's scale factor is the number of bits kept; all of them
|
||||||
|
// means the filter stored the chunk as it was.
|
||||||
|
if !is_float && (cd[1] as i32).max(0) as usize > full_bits {
|
||||||
|
return Err(err("minimum number of bits exceeds maximum"));
|
||||||
|
}
|
||||||
|
if !is_float && cd[1] as i32 == full_bits as i32 {
|
||||||
|
return Ok(data.to_vec());
|
||||||
}
|
}
|
||||||
// The decoded output must match the chunk's uncompressed size; reject an
|
// The decoded output must match the chunk's uncompressed size; reject an
|
||||||
// element count that would over-allocate (e.g. minbits == 0 with a huge
|
// element count that would over-allocate (e.g. minbits == 0 with a huge
|
||||||
// nelmts and no packed payload to bound it).
|
// nelmts and no packed payload to bound it).
|
||||||
let out_bytes = nelmts
|
let out_bytes = nelmts
|
||||||
.checked_mul(elem_size)
|
.checked_mul(elem_size)
|
||||||
.ok_or_else(|| FormatError::ChunkedReadError("scale-offset: size overflow".into()))?;
|
.ok_or_else(|| err("size overflow"))?;
|
||||||
if expected_bytes != 0 && out_bytes > expected_bytes {
|
if expected_bytes != 0 && out_bytes > expected_bytes {
|
||||||
return Err(FormatError::ChunkedReadError(
|
return Err(err("element count exceeds chunk size"));
|
||||||
"scale-offset: element count exceeds chunk size".into(),
|
|
||||||
));
|
|
||||||
}
|
}
|
||||||
let signed = cd[5] == 1;
|
let signed = cd[5] == 1;
|
||||||
let big_endian = cd[6] == 1;
|
let big_endian = cd[6] == 1;
|
||||||
let fill_defined = cd[7] == 1;
|
let fill_defined = cd[7] == 1;
|
||||||
|
|
||||||
// --- header: minbits, then minval, then 8 reserved bytes ---
|
// --- header: minbits, then the size of minval and minval ---
|
||||||
if data.len() < 5 {
|
if data.len() < 5 {
|
||||||
return Err(FormatError::ChunkedReadError(
|
return Err(err("buffer too short"));
|
||||||
"scale-offset: truncated header".into(),
|
|
||||||
));
|
|
||||||
}
|
}
|
||||||
let minbits = u32::from_le_bytes([data[0], data[1], data[2], data[3]]) as usize;
|
let minbits = u32::from_le_bytes([data[0], data[1], data[2], data[3]]) as usize;
|
||||||
let minval_width = data[4] as usize;
|
if minbits > full_bits {
|
||||||
let minval_end = 5 + minval_width;
|
return Err(err("minimum number of bits exceeds size of type"));
|
||||||
if data.len() < minval_end {
|
|
||||||
return Err(FormatError::ChunkedReadError(
|
|
||||||
"scale-offset: truncated minval".into(),
|
|
||||||
));
|
|
||||||
}
|
}
|
||||||
let minval_bytes = &data[5..minval_end];
|
let minval_size = usize::from(data[4]).min(8);
|
||||||
|
let minval_bytes = data
|
||||||
|
.get(5..5 + minval_size)
|
||||||
|
.ok_or_else(|| err("buffer too short"))?;
|
||||||
|
let minval = minval_bytes
|
||||||
|
.iter()
|
||||||
|
.rev()
|
||||||
|
.fold(0u64, |acc, &b| (acc << 8) | u64::from(b));
|
||||||
|
|
||||||
// --- unpack the per-element codes (MSB-first), shared by both variants ---
|
// Full precision: the elements follow as they were, little-endian.
|
||||||
if minbits > 64 {
|
if minbits == full_bits {
|
||||||
return Err(FormatError::ChunkedReadError(
|
let raw = data
|
||||||
"scale-offset: implausible minbits".into(),
|
.get(BUF_OFFSET..)
|
||||||
));
|
.and_then(|d| d.get(..out_bytes))
|
||||||
|
.ok_or_else(|| err("buffer too short"))?;
|
||||||
|
let mut out = raw.to_vec();
|
||||||
|
if big_endian {
|
||||||
|
for e in out.chunks_exact_mut(elem_size) {
|
||||||
|
e.reverse();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return Ok(out);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// --- unpack the per-element codes (MSB-first) ---
|
||||||
let codes: Vec<u64> = if minbits == 0 {
|
let codes: Vec<u64> = if minbits == 0 {
|
||||||
// No packed payload: every element equals minval.
|
// No packed payload: every code is 0.
|
||||||
vec![0u64; nelmts]
|
vec![0u64; nelmts]
|
||||||
} else {
|
} else {
|
||||||
let packed = data.get(minval_end + 8..).ok_or_else(|| {
|
let packed = data.get(BUF_OFFSET..).unwrap_or(&[]);
|
||||||
FormatError::ChunkedReadError("scale-offset: truncated packed data".into())
|
|
||||||
})?;
|
|
||||||
let need_bits = nelmts
|
let need_bits = nelmts
|
||||||
.checked_mul(minbits)
|
.checked_mul(minbits)
|
||||||
.ok_or_else(|| FormatError::ChunkedReadError("scale-offset: size overflow".into()))?;
|
.ok_or_else(|| err("size overflow"))?;
|
||||||
if packed.len() * 8 < need_bits {
|
if packed.len() * 8 < need_bits {
|
||||||
return Err(FormatError::ChunkedReadError(
|
return Err(err("packed data too short"));
|
||||||
"scale-offset: packed data too short".into(),
|
|
||||||
));
|
|
||||||
}
|
}
|
||||||
let mut out = Vec::with_capacity(nelmts);
|
let mut out = Vec::with_capacity(nelmts);
|
||||||
let mut bitpos = 0usize;
|
let mut bitpos = 0usize;
|
||||||
@@ -408,35 +444,28 @@ fn scaleoffset_decompress(
|
|||||||
}
|
}
|
||||||
out
|
out
|
||||||
};
|
};
|
||||||
// The fill code (all ones) only exists when there are bits to pack.
|
// With a fill value defined, the all-ones code of `minbits` bits (0
|
||||||
let has_fill_code = fill_defined && minbits > 0 && minbits < 64;
|
// when minbits is 0) stands for it. minbits < 64 here.
|
||||||
// Computed for all 1..=64 widths; `1 << 64` would overflow, so saturate.
|
let fill_code: u64 = (1u64 << minbits) - 1;
|
||||||
let fill_code: u64 = if minbits == 0 {
|
let fill_bits = || {
|
||||||
0
|
let lo = u64::from(*cd.get(8).unwrap_or(&0));
|
||||||
} else if minbits >= 64 {
|
let hi = u64::from(*cd.get(9).unwrap_or(&0));
|
||||||
u64::MAX
|
lo | (hi << 32)
|
||||||
} else {
|
|
||||||
(1u64 << minbits) - 1
|
|
||||||
};
|
};
|
||||||
|
|
||||||
if is_float {
|
if is_float {
|
||||||
let is_escale = scale_type == H5Z_SO_FLOAT_ESCALE;
|
|
||||||
let scale_factor = cd[1] as i32;
|
let scale_factor = cd[1] as i32;
|
||||||
let minval = read_le_float(minval_bytes, elem_size);
|
let minval = bits_to_float(minval, elem_size);
|
||||||
let fill_value = if fill_defined {
|
let fill_value = if fill_defined {
|
||||||
let lo = *cd.get(8).unwrap_or(&0) as u64;
|
bits_to_float(fill_bits(), elem_size)
|
||||||
let hi = *cd.get(9).unwrap_or(&0) as u64;
|
|
||||||
bits_to_float(lo | (hi << 32), elem_size)
|
|
||||||
} else {
|
} else {
|
||||||
0.0
|
0.0
|
||||||
};
|
};
|
||||||
let values: Vec<f64> = codes
|
let values: Vec<f64> = codes
|
||||||
.iter()
|
.iter()
|
||||||
.map(|&code| {
|
.map(|&code| {
|
||||||
if has_fill_code && code == fill_code {
|
if fill_defined && code == fill_code {
|
||||||
fill_value
|
fill_value
|
||||||
} else if is_escale {
|
|
||||||
minval + code as f64 * powi_f64(2.0, scale_factor)
|
|
||||||
} else if elem_size == 4 {
|
} else if elem_size == 4 {
|
||||||
// H5Z_scaleoffset_modify_3/4 for `float`: the code is
|
// H5Z_scaleoffset_modify_3/4 for `float`: the code is
|
||||||
// read as an `int` and everything is single precision,
|
// read as an `int` and everything is single precision,
|
||||||
@@ -456,21 +485,19 @@ fn scaleoffset_decompress(
|
|||||||
.collect();
|
.collect();
|
||||||
Ok(write_floats(&values, elem_size, big_endian))
|
Ok(write_floats(&values, elem_size, big_endian))
|
||||||
} else {
|
} else {
|
||||||
let minval = read_le_int(minval_bytes, signed);
|
|
||||||
let fill_value: i64 = if fill_defined {
|
let fill_value: i64 = if fill_defined {
|
||||||
let lo = *cd.get(8).unwrap_or(&0) as u64;
|
sign_extend(fill_bits(), elem_size, signed)
|
||||||
let hi = *cd.get(9).unwrap_or(&0) as u64;
|
|
||||||
sign_extend(lo | (hi << 32), elem_size, signed)
|
|
||||||
} else {
|
} else {
|
||||||
0
|
0
|
||||||
};
|
};
|
||||||
let values: Vec<i64> = codes
|
let values: Vec<i64> = codes
|
||||||
.iter()
|
.iter()
|
||||||
.map(|&code| {
|
.map(|&code| {
|
||||||
if has_fill_code && code == fill_code {
|
if fill_defined && code == fill_code {
|
||||||
fill_value
|
fill_value
|
||||||
} else {
|
} else {
|
||||||
minval.wrapping_add(code as i64)
|
// `(type)(buf[i] + minval)`: wraps at the element width.
|
||||||
|
(code.wrapping_add(minval)) as i64
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
@@ -478,21 +505,6 @@ fn scaleoffset_decompress(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Read a little-endian float of `size` bytes (4 = f32, otherwise f64) as f64.
|
|
||||||
fn read_le_float(bytes: &[u8], size: usize) -> f64 {
|
|
||||||
if size == 4 {
|
|
||||||
let mut b = [0u8; 4];
|
|
||||||
let n = bytes.len().min(4);
|
|
||||||
b[..n].copy_from_slice(&bytes[..n]);
|
|
||||||
f32::from_le_bytes(b) as f64
|
|
||||||
} else {
|
|
||||||
let mut b = [0u8; 8];
|
|
||||||
let n = bytes.len().min(8);
|
|
||||||
b[..n].copy_from_slice(&bytes[..n]);
|
|
||||||
f64::from_le_bytes(b)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Interpret the low bits of `raw` as an IEEE float of `size` bytes.
|
/// Interpret the low bits of `raw` as an IEEE float of `size` bytes.
|
||||||
fn bits_to_float(raw: u64, size: usize) -> f64 {
|
fn bits_to_float(raw: u64, size: usize) -> f64 {
|
||||||
if size == 4 {
|
if size == 4 {
|
||||||
@@ -525,16 +537,6 @@ fn write_floats(values: &[f64], elem_size: usize, big_endian: bool) -> Vec<u8> {
|
|||||||
out
|
out
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Read a little-endian integer of `bytes.len()` bytes, sign-extending when
|
|
||||||
/// `signed`. Used for the scale-offset `minval` field.
|
|
||||||
fn read_le_int(bytes: &[u8], signed: bool) -> i64 {
|
|
||||||
let mut raw: u64 = 0;
|
|
||||||
for (i, &b) in bytes.iter().enumerate().take(8) {
|
|
||||||
raw |= (b as u64) << (i * 8);
|
|
||||||
}
|
|
||||||
sign_extend(raw, bytes.len().min(8), signed)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Interpret the low `size` bytes of `raw` as a (possibly signed) integer.
|
/// Interpret the low `size` bytes of `raw` as a (possibly signed) integer.
|
||||||
fn sign_extend(raw: u64, size: usize, signed: bool) -> i64 {
|
fn sign_extend(raw: u64, size: usize, signed: bool) -> i64 {
|
||||||
if size == 0 || size >= 8 {
|
if size == 0 || size >= 8 {
|
||||||
@@ -1216,6 +1218,23 @@ fn zstd_compress(data: &[u8], level: u32) -> Result<Vec<u8>, FormatError> {
|
|||||||
/// Unshuffle (decompress direction): reconstruct interleaved element bytes.
|
/// Unshuffle (decompress direction): reconstruct interleaved element bytes.
|
||||||
/// On disk: all byte-0s of each element together, then all byte-1s, etc.
|
/// On disk: all byte-0s of each element together, then all byte-1s, etc.
|
||||||
/// Output: elements in natural order.
|
/// Output: elements in natural order.
|
||||||
|
/// The element size the shuffle filter works with: its parameter, as
|
||||||
|
/// libhdf5 uses it (`H5Z__filter_shuffle`), not the dataset's element size.
|
||||||
|
/// They are the same in every file a library wrote; a corrupt parameter
|
||||||
|
/// larger than the chunk makes libhdf5 leave the chunk as it is, and so
|
||||||
|
/// does [`shuffle_decompress`] (`cve-2025-44905`'s `Shuffle_float_data_be`).
|
||||||
|
/// A zero parameter is an error ("invalid shuffle parameters"); a pipeline
|
||||||
|
/// without the parameter (never written by libhdf5) uses the element size.
|
||||||
|
fn shuffle_type_size(cd: &[u32], element_size: usize) -> Result<usize, FormatError> {
|
||||||
|
match cd {
|
||||||
|
[] => Ok(element_size),
|
||||||
|
[0] | [_, _, ..] => Err(FormatError::FilterError(
|
||||||
|
"invalid shuffle parameters".into(),
|
||||||
|
)),
|
||||||
|
[size] => Ok(*size as usize),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn shuffle_decompress(data: &[u8], element_size: usize) -> Result<Vec<u8>, FormatError> {
|
fn shuffle_decompress(data: &[u8], element_size: usize) -> Result<Vec<u8>, FormatError> {
|
||||||
if element_size <= 1 {
|
if element_size <= 1 {
|
||||||
return Ok(data.to_vec());
|
return Ok(data.to_vec());
|
||||||
@@ -2434,48 +2453,125 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn as_f64(bytes: &[u8]) -> Vec<f64> {
|
/// E-scale: libhdf5 refuses it on read and write ("E-scaling method not
|
||||||
bytes
|
/// supported"); it was decoded here, never checked against anything.
|
||||||
.as_chunks::<8>()
|
#[test]
|
||||||
.0
|
fn scaleoffset_float_escale_is_refused_as_in_libhdf5() {
|
||||||
.iter()
|
let cd = [1u32, 1, 4, 1, 8, 0, 0, 0];
|
||||||
.map(|c| f64::from_le_bytes(*c))
|
let mut raw = vec![2, 0, 0, 0, 8];
|
||||||
.collect()
|
raw.extend_from_slice(&[0; 16]);
|
||||||
|
raw.push(0x1B);
|
||||||
|
assert!(scaleoffset_decompress(&raw, &cd, 0).is_err());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A scale type that does not match the class is refused, as libhdf5
|
||||||
|
/// refuses it ("invalid scale type").
|
||||||
#[test]
|
#[test]
|
||||||
fn scaleoffset_float_escale_e1() {
|
fn scaleoffset_scale_type_must_match_the_class() {
|
||||||
// f64 [0.0, 2.0, 4.0, 6.0], E=1 (×2^1=2), fill_defined=0.
|
let mut raw = vec![2, 0, 0, 0, 8];
|
||||||
// cd: scale_type=1, E=1, nelmts=4, elem_size=8.
|
raw.extend_from_slice(&[0; 16]);
|
||||||
let cd = [1u32, 1, 4, 0, 8, 0, 0, 0];
|
raw.push(0x1B);
|
||||||
let raw: &[u8] = &[
|
assert!(scaleoffset_decompress(&raw, &[0, 0, 4, 0, 4, 1, 0, 0], 0).is_err());
|
||||||
2, 0, 0, 0, // minbits=2
|
assert!(scaleoffset_decompress(&raw, &[2, 0, 4, 1, 4, 1, 0, 0], 0).is_err());
|
||||||
8, // minval_width=8
|
assert!(scaleoffset_decompress(&raw, &[2, 0, 4, 7, 4, 1, 0, 0], 0).is_err());
|
||||||
0, 0, 0, 0, 0, 0, 0, 0, // minval=0.0f64
|
|
||||||
0, 0, 0, 0, 0, 0, 0, 0, // 8 reserved bytes
|
|
||||||
0x1B, // packed codes: 00 01 10 11 MSB-first
|
|
||||||
];
|
|
||||||
let got = as_f64(&scaleoffset_decompress(raw, &cd, 0).unwrap());
|
|
||||||
assert_eq!(got, vec![0.0, 2.0, 4.0, 6.0]);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// `cve-2025-44905` `/Scale_offset_short_data_be`, chunk (4, 0): the
|
||||||
|
/// stored size of `minval` is 0. libhdf5 reads a `minval` of 0 and the
|
||||||
|
/// packed codes from byte 21 regardless; we read them from byte 13
|
||||||
|
/// (5 + size + 8), so the values differed from h5py's.
|
||||||
#[test]
|
#[test]
|
||||||
fn scaleoffset_float_escale_neg_exp() {
|
fn scaleoffset_codes_start_at_byte_21_whatever_the_minval_size() {
|
||||||
// f64 [0.0, 0.5, 1.0, 1.5], E=-1 (×2^-1=0.5), fill_defined=0.
|
// big-endian i16, 12 elements, fill -2 (cd 65534), minbits 3.
|
||||||
// cd[1] = 0xFFFF_FFFF which casts to i32 = -1.
|
let mut cd = vec![2u32, 0, 12, 0, 2, 1, 1, 1, 65534];
|
||||||
let cd = [1u32, 0xFFFF_FFFF, 4, 0, 8, 0, 0, 0];
|
cd.resize(20, 0);
|
||||||
let raw: &[u8] = &[
|
let raw = unhex("0300000000d20e00000000000034000000000000000400000000");
|
||||||
2, 0, 0, 0, // minbits=2
|
let got = scaleoffset_decompress(&raw, &cd, 24).unwrap();
|
||||||
8, // minval_width=8
|
// Codes of 3 bits from byte 21 (04 00 00 00 00): 0, 1, 0, ...; h5py
|
||||||
0, 0, 0, 0, 0, 0, 0, 0, // minval=0.0f64
|
// reads the chunk's first row as 0, 1, 0.
|
||||||
0, 0, 0, 0, 0, 0, 0, 0, // 8 reserved bytes
|
let want: Vec<i16> = vec![0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0];
|
||||||
0x1B, // packed codes: 00 01 10 11 MSB-first
|
let want: Vec<u8> = want.iter().flat_map(|v| v.to_be_bytes()).collect();
|
||||||
];
|
assert_eq!(got, want);
|
||||||
let got = as_f64(&scaleoffset_decompress(raw, &cd, 0).unwrap());
|
}
|
||||||
let exp = [0.0f64, 0.5, 1.0, 1.5];
|
|
||||||
for (g, e) in got.iter().zip(exp.iter()) {
|
/// With a fill value defined, libhdf5 compares each code with the
|
||||||
assert!((g - e).abs() < 1e-9, "got {g} expected {e}");
|
/// all-ones code of `minbits` bits — which for `minbits == 0` is 0, so a
|
||||||
}
|
/// chunk with no packed codes reads as all fill values (the compressor
|
||||||
|
/// writes that for a chunk of nothing but fill values). It read as
|
||||||
|
/// `minval` here.
|
||||||
|
#[test]
|
||||||
|
fn scaleoffset_minbits_zero_with_a_fill_value_is_all_fill() {
|
||||||
|
let mut cd = vec![2u32, 0, 3, 0, 4, 1, 0, 1, (-7i32) as u32];
|
||||||
|
cd.resize(20, 0);
|
||||||
|
let mut raw = vec![0, 0, 0, 0, 8];
|
||||||
|
raw.extend_from_slice(&5i64.to_le_bytes());
|
||||||
|
raw.extend_from_slice(&[0; 8]);
|
||||||
|
assert_eq!(
|
||||||
|
scaleoffset_decompress(&raw, &cd, 12).unwrap(),
|
||||||
|
i32_le(&[-7, -7, -7])
|
||||||
|
);
|
||||||
|
// Without a fill value every element is minval.
|
||||||
|
cd[7] = 0;
|
||||||
|
assert_eq!(
|
||||||
|
scaleoffset_decompress(&raw, &cd, 12).unwrap(),
|
||||||
|
i32_le(&[5, 5, 5])
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `minbits` of the full width stores the elements as they are
|
||||||
|
/// (little-endian), without `minval`; a full-width integer scale factor
|
||||||
|
/// means the filter left the chunk untouched.
|
||||||
|
#[test]
|
||||||
|
fn scaleoffset_full_width_is_stored_as_is() {
|
||||||
|
let mut cd = vec![2u32, 0, 2, 0, 2, 1, 1, 0];
|
||||||
|
cd.resize(20, 0);
|
||||||
|
let mut raw = vec![16, 0, 0, 0, 8];
|
||||||
|
raw.extend_from_slice(&100i64.to_le_bytes());
|
||||||
|
raw.extend_from_slice(&[0; 8]);
|
||||||
|
raw.extend_from_slice(&[0x34, 0x12, 0xfe, 0xff]);
|
||||||
|
assert_eq!(
|
||||||
|
scaleoffset_decompress(&raw, &cd, 4).unwrap(),
|
||||||
|
[0x12, 0x34, 0xff, 0xfe]
|
||||||
|
);
|
||||||
|
cd[1] = 16;
|
||||||
|
assert_eq!(
|
||||||
|
scaleoffset_decompress(&[1, 2, 3, 4], &cd, 4).unwrap(),
|
||||||
|
[1, 2, 3, 4]
|
||||||
|
);
|
||||||
|
cd[1] = 17;
|
||||||
|
assert!(scaleoffset_decompress(&[1, 2, 3, 4], &cd, 4).is_err());
|
||||||
|
// minbits wider than the type.
|
||||||
|
cd[1] = 0;
|
||||||
|
raw[0] = 17;
|
||||||
|
assert!(scaleoffset_decompress(&raw, &cd, 4).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The shuffle filter uses its own parameter as the element size, as
|
||||||
|
/// libhdf5 does; a parameter larger than the chunk leaves the chunk as
|
||||||
|
/// it is (`cve-2025-44905` `/Shuffle_float_data_be`, whose parameter is
|
||||||
|
/// 4261347332: h5py and h5dump read the stored bytes unshuffled).
|
||||||
|
#[test]
|
||||||
|
fn shuffle_uses_its_parameter() {
|
||||||
|
let data: Vec<u8> = (0..16).collect();
|
||||||
|
let shuffled = shuffle_compress(&data, 4).unwrap();
|
||||||
|
let pipeline = |cd: Vec<u32>| FilterPipeline {
|
||||||
|
version: 2,
|
||||||
|
filters: vec![one_filter(FILTER_SHUFFLE, cd)],
|
||||||
|
};
|
||||||
|
// The dataset's element size says 2; the parameter says 4.
|
||||||
|
assert_eq!(
|
||||||
|
decompress_chunk(&shuffled, &pipeline(vec![4]), 16, 2).unwrap(),
|
||||||
|
data
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
decompress_chunk(&shuffled, &pipeline(vec![4_261_347_332]), 16, 4).unwrap(),
|
||||||
|
shuffled
|
||||||
|
);
|
||||||
|
assert!(decompress_chunk(&shuffled, &pipeline(vec![0]), 16, 4).is_err());
|
||||||
|
assert_eq!(
|
||||||
|
decompress_chunk(&shuffled, &pipeline(vec![]), 16, 4).unwrap(),
|
||||||
|
data
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// --- N-Bit (filter id 5) --------------------------------------------------
|
// --- N-Bit (filter id 5) --------------------------------------------------
|
||||||
|
|||||||
Reference in New Issue
Block a user