fix: refuse at open the dataset storage libhdf5 refuses at open

libhdf5 checks a dataset's storage when it opens the dataset
(H5D__contig_check, H5D__compact_init): the element count times the
element size must not overflow, contiguous storage must end inside the
file, compact data must be the dataset's size. File::dataset opened
cve-2024-32624's /Dset_OBJREF (2^62 + 2 references of 8 bytes) and
reported its shape; only reading failed.

data_read::check_dataset_storage makes those checks (new
FormatError::InvalidDatasetStorage), and File, MmapFile and LazyFile run
it whenever they open a dataset (by path, by address, from a group), as
does the conformance probe. As before, a datatype, dataspace or layout
that does not decode is left for the read to report, so such a dataset
still opens and its attributes still read. An empty contiguous dataset at
a defined address, which libhdf5 refuses, is still accepted: clawhdf5 up
to v2.7.0 wrote them.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>

crates/clawhdf5/src/lazy.rs

@@ -135,10 +135,11 @@ impl<R: HDF5Read> LazyFile<R> {
         if !has_message(&hdr, MessageType::DataLayout) {
             return Err(Error::NotADataset(path.to_string()));
         }
-        Ok(LazyDataset {
+        LazyDataset {
             file: self,
             header: hdr,
-        })
+        }
+        .check_open()
     }
 
     /// Resolve a path and return a `LazyGroup` handle.
@@ -284,10 +285,11 @@ impl<'f, R: HDF5Read> LazyGroup<'f, R> {
         if !has_message(&hdr, MessageType::DataLayout) {
             return Err(Error::NotADataset(name.to_string()));
         }
-        Ok(LazyDataset {
+        LazyDataset {
             file: self.file,
             header: hdr,
-        })
+        }
+        .check_open()
     }
 
     /// Get a subgroup within this group by name.
@@ -326,6 +328,24 @@ pub struct LazyDataset<'f, R: HDF5Read> {
 }
 
 impl<'f, R: HDF5Read> LazyDataset<'f, R> {
+    /// libhdf5's storage checks when it opens a dataset
+    /// ([`data_read::check_dataset_storage`]): a dataset whose element count
+    /// times element size overflows, or whose contiguous storage runs past
+    /// the end of the file, fails to open. A datatype, dataspace or layout
+    /// that does not decode is left for the read to report, as before (the
+    /// dataset still opens, and its attributes can be read).
+    fn check_open(self) -> Result<Self, Error> {
+        let decoded = (|| -> Result<_, Error> {
+            let data = self.required_payload(MessageType::Dataspace)?;
+            let ds = Dataspace::parse(&data, self.file.length_size())?;
+            Ok((self.datatype()?, ds, self.data_layout()?))
+        })();
+        if let Ok((dt, ds, dl)) = decoded {
+            data_read::check_dataset_storage(&dl, &ds, &dt, self.file.hdf5_bytes().len() as u64)?;
+        }
+        Ok(self)
+    }
+
     /// Returns the shape (dimensions) of the dataset.
     pub fn shape(&self) -> Result<Vec<u64>, Error> {
         let ds = self.dataspace()?;

crates/clawhdf5/src/mmap_file.rs

@@ -85,10 +85,11 @@ impl MmapFile {
         if !has_message(&hdr, MessageType::DataLayout) {
             return Err(Error::NotADataset(path.to_string()));
         }
-        Ok(MmapDataset {
+        MmapDataset {
             file: self,
             header: hdr,
-        })
+        }
+        .check_open()
     }
 
     /// Resolve a path and return a `MmapGroup` handle.
@@ -211,10 +212,11 @@ impl<'f> MmapGroup<'f> {
         if !has_message(&hdr, MessageType::DataLayout) {
             return Err(Error::NotADataset(name.to_string()));
         }
-        Ok(MmapDataset {
+        MmapDataset {
             file: self.file,
             header: hdr,
-        })
+        }
+        .check_open()
     }
 
     /// Get a subgroup within this group by name.
@@ -253,6 +255,24 @@ pub struct MmapDataset<'f> {
 }
 
 impl<'f> MmapDataset<'f> {
+    /// libhdf5's storage checks when it opens a dataset
+    /// ([`data_read::check_dataset_storage`]): a dataset whose element count
+    /// times element size overflows, or whose contiguous storage runs past
+    /// the end of the file, fails to open. A datatype, dataspace or layout
+    /// that does not decode is left for the read to report, as before (the
+    /// dataset still opens, and its attributes can be read).
+    fn check_open(self) -> Result<Self, Error> {
+        let decoded = (|| -> Result<_, Error> {
+            let data = self.required_payload(MessageType::Dataspace)?;
+            let ds = Dataspace::parse(&data, self.file.length_size())?;
+            Ok((self.datatype()?, ds, self.data_layout()?))
+        })();
+        if let Ok((dt, ds, dl)) = decoded {
+            data_read::check_dataset_storage(&dl, &ds, &dt, self.file.hdf5_bytes().len() as u64)?;
+        }
+        Ok(self)
+    }
+
     /// Returns the shape (dimensions) of the dataset.
     pub fn shape(&self) -> Result<Vec<u64>, Error> {
         let ds = self.dataspace()?;

crates/clawhdf5/src/reader.rs

@@ -170,10 +170,11 @@ impl File {
         if !has_message(&hdr, MessageType::DataLayout) {
             return Err(Error::NotADataset(path.to_string()));
         }
-        Ok(Dataset {
+        Dataset {
             file: self,
             header: hdr,
-        })
+        }
+        .check_open()
     }
 
     /// A `Dataset` handle for the object header at `address` (an address
@@ -186,10 +187,11 @@ impl File {
         if !has_message(&hdr, MessageType::DataLayout) {
             return Err(Error::NotADataset(format!("object at address {address}")));
         }
-        Ok(Dataset {
+        Dataset {
             file: self,
             header: hdr,
-        })
+        }
+        .check_open()
     }
 
     /// Resolve a path and return a `Group` handle.
@@ -427,10 +429,11 @@ impl<'f> Group<'f> {
         if !has_message(&hdr, MessageType::DataLayout) {
             return Err(Error::NotADataset(name.to_string()));
         }
-        Ok(Dataset {
+        Dataset {
             file: self.file,
             header: hdr,
-        })
+        }
+        .check_open()
     }
 
     /// Get a subgroup within this group by name.
@@ -469,6 +472,24 @@ pub struct Dataset<'f> {
 }
 
 impl<'f> Dataset<'f> {
+    /// libhdf5's storage checks when it opens a dataset
+    /// ([`data_read::check_dataset_storage`]): a dataset whose element count
+    /// times element size overflows, or whose contiguous storage runs past
+    /// the end of the file, fails to open. A datatype, dataspace or layout
+    /// that does not decode is left for the read to report, as before (the
+    /// dataset still opens, and its attributes can be read).
+    fn check_open(self) -> Result<Self, Error> {
+        let decoded = (|| -> Result<_, Error> {
+            let data = self.required_payload(MessageType::Dataspace)?;
+            let ds = Dataspace::parse(&data, self.file.length_size())?;
+            Ok((self.datatype()?, ds, self.data_layout()?))
+        })();
+        if let Ok((dt, ds, dl)) = decoded {
+            data_read::check_dataset_storage(&dl, &ds, &dt, self.file.data.len() as u64)?;
+        }
+        Ok(self)
+    }
+
     /// Returns the shape (dimensions) of the dataset.
     pub fn shape(&self) -> Result<Vec<u64>, Error> {
         let ds = self.dataspace()?;

crates/clawhdf5/tests/header_validation_interop.rs

@@ -508,3 +508,52 @@ for libver in ("earliest", "latest"):
         );
     }
 }
+
+/// cve-2024-32624 `/Dset_OBJREF`: a dataspace whose element count times the
+/// element size overflows 64 bits. libhdf5 refuses to open the dataset
+/// ("size of dataset's storage overflowed"); `File::dataset` used to open it
+/// and report its shape, and only reading failed. The same for storage that
+/// runs past the end of the file ("invalid dataset size, likely file
+/// corruption").
+#[test]
+fn dataset_storage_libhdf5_refuses_at_open_is_refused_at_open() {
+    skip_if_no_python!();
+    let dir = tempfile::tempdir().unwrap();
+    // A contiguous int64 dataset of 3 elements, libver earliest (a version 1
+    // dataspace holding the size and the maximum size). "overflow" makes
+    // both 2^62 + 2 (x 8 bytes overflows); "pasteof" makes both 1000 (the
+    // storage runs past the end of the file).
+    let verdicts = h5py_verdicts(
+        dir.path(),
+        r#"
+good = os.path.join(d, "good.h5")
+with h5py.File(good, "w", libver="earliest") as f:
+    f.create_dataset("d", data=np.array([7, 8, 9], dtype="<i8"))
+data = bytearray(open(good, "rb").read())
+three = struct.pack("<QQ", 3, 3)
+at = data.find(three)
+assert at > 0 and data.find(three, at + 1) < 0
+for name, n in (("overflow", (1 << 62) + 2), ("pasteof", 1000)):
+    bad = bytearray(data)
+    bad[at:at + 16] = struct.pack("<QQ", n, n)
+    open(os.path.join(d, f"{name}.h5"), "wb").write(bad)
+"#,
+    );
+    assert_agrees_with_h5py(
+        dir.path(),
+        &verdicts,
+        &["good ok", "overflow ERROR", "pasteof ERROR"],
+    );
+    for name in ["overflow", "pasteof"] {
+        let path = dir.path().join(format!("{name}.h5"));
+        let file = File::open(&path).unwrap();
+        let err = file.dataset("d").unwrap_err();
+        assert!(
+            err.to_string().contains("invalid dataset storage"),
+            "{name}: {err}"
+        );
+        assert!(file.root().dataset("d").is_err(), "{name}");
+        let mm = clawhdf5::MmapFile::open(&path).unwrap();
+        assert!(mm.dataset("d").is_err(), "{name}: MmapFile");
+    }
+}