fix: refuse numeric types with unusually many unused bits in v1 headers

libhdf5 1.14.4+ treats an integer, float or bit field wider than a byte
whose precision and offset leave more than half its bits unused as
corruption when the type sits in a header without a checksum (version 1),
unless the file is opened with H5Pset_relax_file_integrity_checks
(H5T_is_numeric_with_unusual_unused_bits). clawhdf5 read such types,
e.g. a 3-bit integer in 4 bytes (cve-2024-29162) or a 32-bit float in
65525 bytes (cve-2024-32614, tmisc38a.h5).

New Datatype::check_unused_bits (recursive) and Datatype::parse_in_header,
which applies it for version-1 headers. Dataset datatypes (facade File,
LazyFile, MmapFile; clawhdf5-io VOL, MPI VOL, async reader; the
conformance probe) and compact attributes in version-1 headers use it.

Conformance (cached corpus, tank): 570 ok, unchanged; cve-2024-29162,
cve-2024-32614 and tmisc38a.h5 now refuse the object h5py refuses, and
tmisc38b.h5 / unknown-1.h5 now fail with libhdf5's reason.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>

crates/clawhdf5-format/src/attribute.rs

@@ -362,6 +362,18 @@ fn extract_name(bytes: &[u8]) -> String {
     String::from_utf8_lossy(&bytes[..end]).into_owned()
 }
 
+/// An attribute's datatype gets libhdf5's extra check for a header without
+/// a checksum (see [`Datatype::check_unused_bits`]).
+fn check_in_header(
+    attr: AttributeMessage,
+    header: &ObjectHeader,
+) -> Result<AttributeMessage, FormatError> {
+    if header.version == 1 {
+        attr.datatype.check_unused_bits()?;
+    }
+    Ok(attr)
+}
+
 /// Extract all attribute messages from an object header.
 pub fn extract_attributes(
     header: &ObjectHeader,
@@ -371,7 +383,7 @@ pub fn extract_attributes(
     for msg in &header.messages {
         if msg.msg_type == MessageType::Attribute {
             let attr = AttributeMessage::parse(&msg.data, length_size)?;
-            attrs.push(attr);
+            attrs.push(check_in_header(attr, header)?);
         }
     }
     Ok(attrs)
@@ -465,6 +477,7 @@ fn extract_attributes_with(
             } else {
                 AttributeMessage::parse_in_file(&msg.data, file_data, offset_size, length_size)
             };
+            let attr = attr.and_then(|a| check_in_header(a, header));
             match attr {
                 Ok(attr) => attrs.push(attr),
                 Err(e) => on_error(e)?,

crates/clawhdf5-format/src/datatype.rs

@@ -858,6 +858,70 @@ impl Datatype {
         }
     }
 
+    /// [`Self::parse`] for the datatype message of an object whose header
+    /// has version `header_version`: a version-1 header, which has no
+    /// checksum, additionally gets [`Self::check_unused_bits`], as libhdf5
+    /// does. Use this wherever the header is at hand.
+    pub fn parse_in_header(
+        data: &[u8],
+        header_version: u8,
+    ) -> Result<(Datatype, usize), FormatError> {
+        let parsed = Self::parse(data)?;
+        if header_version == 1 {
+            parsed.0.check_unused_bits()?;
+        }
+        Ok(parsed)
+    }
+
+    /// libhdf5's guard against a corrupt numeric type in a header without
+    /// a checksum (`H5T_is_numeric_with_unusual_unused_bits`, HDF5 1.14.4+):
+    /// an integer, float or bit field wider than a byte whose precision and
+    /// offset leave more than half its bits unused is taken for corruption
+    /// (e.g. a 3-bit integer in 4 bytes, `cve-2024-29162`, or a 32-bit float
+    /// in 65525 bytes, `cve-2024-32614`), anywhere in the type. libhdf5
+    /// skips the check for checksummed (version-2) headers and when the
+    /// file is opened with `H5Pset_relax_file_integrity_checks`; so does
+    /// [`Self::parse_in_header`], which has no such option.
+    pub fn check_unused_bits(&self) -> Result<(), FormatError> {
+        match self {
+            Datatype::FixedPoint {
+                size,
+                bit_offset,
+                bit_precision,
+                ..
+            }
+            | Datatype::FloatingPoint {
+                size,
+                bit_offset,
+                bit_precision,
+                ..
+            }
+            | Datatype::BitField {
+                size,
+                bit_offset,
+                bit_precision,
+                ..
+            } => {
+                let bits = u64::from(*size) * 8;
+                let prec = u64::from(*bit_precision);
+                if *size > 1 && prec < bits && bits > 2 * (prec + u64::from(*bit_offset)) {
+                    return Err(invalid(format!(
+                        "datatype has unusually large # of unused bits (prec = {prec} bits, \
+                         size = {size} bytes), possibly corrupted file"
+                    )));
+                }
+                Ok(())
+            }
+            Datatype::Compound { members, .. } => members
+                .iter()
+                .try_for_each(|m| m.datatype.check_unused_bits()),
+            Datatype::Enumeration { base_type, .. }
+            | Datatype::VariableLength { base_type, .. }
+            | Datatype::Array { base_type, .. } => base_type.check_unused_bits(),
+            _ => Ok(()),
+        }
+    }
+
     /// Serialize datatype to HDF5 message bytes.
     pub fn serialize(&self) -> Vec<u8> {
         match self {
@@ -2354,6 +2418,35 @@ mod tests {
         assert_eq!(invalid_reason(&data), "bad byte order for datatype message");
     }
 
+    #[test]
+    fn unusual_unused_bits_are_refused_in_version_1_headers_only() {
+        // cve-2024-29162: a 3-bit integer in 4 bytes.
+        let data = build_fixed_point(4, false, true, 0, 3);
+        assert!(Datatype::parse_in_header(&data, 2).is_ok());
+        assert_eq!(
+            match Datatype::parse_in_header(&data, 1) {
+                Err(FormatError::InvalidDatatype(why)) => why,
+                other => panic!("{other:?}"),
+            },
+            "datatype has unusually large # of unused bits (prec = 3 bits, size = 4 bytes), \
+             possibly corrupted file"
+        );
+        // Half the bits used (with the offset) is not unusual; nor is a
+        // 1-byte type; nor a full-precision one.
+        for (size, offset, prec) in [(4u32, 0u16, 16u16), (4, 8, 8), (1, 0, 1), (8, 0, 64)] {
+            let data = build_fixed_point(size, false, true, offset, prec);
+            assert!(
+                Datatype::parse_in_header(&data, 1).is_ok(),
+                "{size} {offset} {prec}"
+            );
+        }
+        // Nested: a compound member's type is checked too.
+        let member = build_fixed_point(4, false, true, 0, 15);
+        let data = compound_v3(4, &[("a", 0, member)]);
+        assert!(Datatype::parse_in_header(&data, 2).is_ok());
+        assert!(Datatype::parse_in_header(&data, 1).is_err());
+    }
+
     #[test]
     fn float_bit_6_is_vax_order_only_from_version_3() {
         // h5py opens a v1 float with bit 6 set as an ordinary little-endian