feat: decode the superblock extension at open; read metadata cache images

libhdf5 decodes the messages of a v2/v3 superblock's extension when it
opens a file (H5F__super_read) and refuses the file when one does not
decode. We never looked at them, so we opened cve-2020-10810 (a File
Space Info message too short for the free-space manager addresses it
announces) and cve-2020-10812 (a metadata cache image past the end of the
file), both of which libhdf5 refuses.

A file written with a metadata cache image keeps its metadata cache
entries in an image block the extension points at; libhdf5 loads them
over the file's own bytes before it reads any metadata
(H5C__load_cache_image, H5C__reconstruct_cache_contents). In
h5clear_mdc_image.h5 the root group's header exists only in the image, so
every reader failed with InvalidObjectHeaderVersion(0).

The new clawhdf5_format::superblock_ext module:
- read_superblock_extension decodes the v1 B-tree K, File Space Info and
  Metadata Cache Image messages with libhdf5's checks (versions, page size
  512 B .. 1 GiB, the addresses a persisting message lists, the image
  inside the file), with the new FormatError::InvalidSuperblockExtension;
- apply_cache_image checks an image block as libhdf5 does (signature,
  version, recorded length, entry types, rings, ages, addresses inside
  the file and not repeated, flush-dependency parents) and returns the
  file's bytes with every entry written at its address
  (FormatError::InvalidCacheImage);
- metadata_view does both.

File, MmapFile and LazyFile (and so h5rs) call metadata_view at open and
read an image file through the patched copy; the conformance probe does
the same. The image's trailing checksum is not verified, as libhdf5 does
not verify it. tests/fixtures/h5clear_mdc_image.h5 is libhdf5's own test
file.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>

crates/clawhdf5/src/lazy.rs

@@ -46,6 +46,9 @@ pub struct LazyFile<R: HDF5Read> {
     /// End of the HDF5 data (`Superblock::data_end`, absolute).
     end: usize,
     superblock: Superblock,
+    /// The metadata as libhdf5 reads it when the file holds a metadata
+    /// cache image (see `superblock_ext::metadata_view`); `None` otherwise.
+    overlay: Option<Vec<u8>>,
     root_header: ObjectHeader,
     /// Cache of parsed object headers, keyed by address.
     header_cache: RefCell<HashMap<u64, ObjectHeader>>,
@@ -82,7 +85,13 @@ impl<R: HDF5Read> LazyFile<R> {
         let superblock = Superblock::parse(data, 0)?;
         // Refuse a truncated file; read nothing past the recorded end of file.
         let end = base + superblock.data_end(base as u64, whole_len)? as usize;
-        let data = &reader.as_bytes()[base..end];
+        // Decode the superblock extension as libhdf5 does at open, and load
+        // a metadata cache image over the file's metadata.
+        let overlay = clawhdf5_format::superblock_ext::metadata_view(
+            &reader.as_bytes()[base..end],
+            &superblock,
+        )?;
+        let data = overlay.as_deref().unwrap_or(&reader.as_bytes()[base..end]);
         let root_header = ObjectHeader::parse(
             data,
             superblock.root_group_address as usize,
@@ -94,6 +103,7 @@ impl<R: HDF5Read> LazyFile<R> {
             base,
             end,
             superblock,
+            overlay,
             root_header,
             header_cache: RefCell::new(HashMap::new()),
         })
@@ -111,7 +121,10 @@ impl<R: HDF5Read> LazyFile<R> {
     }
 
     fn hdf5_bytes(&self) -> &[u8] {
-        &self.reader.as_bytes()[self.base..self.end]
+        match &self.overlay {
+            Some(v) => v,
+            None => &self.reader.as_bytes()[self.base..self.end],
+        }
     }
 
     /// Returns a reference to the parsed superblock.

crates/clawhdf5/src/mmap_file.rs

@@ -38,6 +38,9 @@ pub struct MmapFile {
     /// End of the HDF5 data (`Superblock::data_end`, absolute).
     end: usize,
     superblock: Superblock,
+    /// The metadata as libhdf5 reads it when the file holds a metadata
+    /// cache image (see `superblock_ext::metadata_view`); `None` otherwise.
+    overlay: Option<Vec<u8>>,
 }
 
 impl MmapFile {
@@ -50,18 +53,28 @@ impl MmapFile {
         let superblock = Superblock::parse(data, 0)?;
         // Refuse a truncated file; read nothing past the recorded end of file.
         let end = base + superblock.data_end(base as u64, whole_len)? as usize;
+        // Decode the superblock extension as libhdf5 does at open, and load
+        // a metadata cache image over the file's metadata.
+        let overlay = clawhdf5_format::superblock_ext::metadata_view(
+            &reader.as_bytes()[base..end],
+            &superblock,
+        )?;
         Ok(Self {
             reader,
             base,
             end,
             superblock,
+            overlay,
         })
     }
 
     /// The file's bytes from the superblock on — the space HDF5 addresses
     /// index into.
     fn hdf5_bytes(&self) -> &[u8] {
-        &self.reader.as_bytes()[self.base..self.end]
+        match &self.overlay {
+            Some(v) => v,
+            None => &self.reader.as_bytes()[self.base..self.end],
+        }
     }
 
     /// Size of the user block before the superblock (0 for most files).

crates/clawhdf5/src/reader.rs

@@ -20,6 +20,7 @@ use clawhdf5_format::message_type::MessageType;
 use clawhdf5_format::object_header::ObjectHeader;
 use clawhdf5_format::signature;
 use clawhdf5_format::superblock::Superblock;
+use clawhdf5_format::superblock_ext;
 
 use crate::error::Error;
 use crate::types::{AttrValue, DType, classify_datatype, read_attrs};
@@ -55,6 +56,11 @@ struct FileData {
     base: usize,
     /// End of the HDF5 data in the file (`Superblock::data_end`, absolute).
     end: usize,
+    /// The file's metadata as libhdf5 reads it when the file holds a
+    /// metadata cache image: the bytes from the superblock to the end of
+    /// file with the image's entries written in
+    /// ([`superblock_ext::metadata_view`]). `None` for every other file.
+    overlay: Option<Vec<u8>>,
 }
 
 impl FileData {
@@ -68,11 +74,26 @@ impl FileData {
         let end = superblock.data_end(base as u64, whole.len() as u64)?;
         // data_end is at most the file length (less the user block).
         let end = base + end as usize;
-        Ok((Self { backing, base, end }, superblock))
+        // libhdf5 decodes the superblock extension at open (a message it
+        // cannot decode fails the open) and loads a metadata cache image
+        // over the file's own metadata.
+        let overlay = superblock_ext::metadata_view(&whole[base..end], &superblock)?;
+        Ok((
+            Self {
+                backing,
+                base,
+                end,
+                overlay,
+            },
+            superblock,
+        ))
     }
 
     fn as_bytes(&self) -> &[u8] {
-        &self.backing.whole_file()[self.base..self.end]
+        match &self.overlay {
+            Some(v) => v,
+            None => &self.backing.whole_file()[self.base..self.end],
+        }
     }
 
     fn len(&self) -> usize {

crates/clawhdf5/tests/header_validation_interop.rs

@@ -42,8 +42,9 @@ macro_rules! skip_if_no_python {
 
 /// Runs `body` (Python, with `h5py`, `numpy as np`, `struct` imported and
 /// `d` the output directory) and then, for every `NAME.h5` it wrote,
-/// prints `NAME ok` when h5py opens and reads dataset `d` and `NAME ERROR`
-/// otherwise. Returns those lines, sorted.
+/// prints `NAME ok` when h5py opens and reads dataset `d` (or the dataset
+/// the body names in `DSET`) and `NAME ERROR` otherwise. Returns those
+/// lines, sorted.
 fn h5py_verdicts(dir: &Path, body: &str) -> Vec<String> {
     let script = format!(
         r#"
@@ -54,7 +55,7 @@ for path in sorted(glob.glob(os.path.join(d, "*.h5"))):
     name = os.path.basename(path)[:-3]
     try:
         with h5py.File(path, "r") as f:
-            f["d"][()]
+            f[globals().get("DSET", "d")][()]
         print(name, "ok")
     except Exception:
         print(name, "ERROR")
@@ -78,14 +79,14 @@ for path in sorted(glob.glob(os.path.join(d, "*.h5"))):
     lines
 }
 
-/// Whether clawhdf5 opens and reads dataset `d` of `path` (as raw bytes of
-/// whatever type it has).
-fn clawhdf5_reads(path: &Path) -> Result<(), String> {
+/// Whether clawhdf5 opens and reads dataset `dset` of `path` (as raw bytes
+/// of whatever type it has).
+fn clawhdf5_reads(path: &Path, dset: &str) -> Result<(), String> {
     let file = File::open(path).map_err(|e| format!("open: {e}"))?;
-    let ds = file.dataset("d").map_err(|e| format!("dataset: {e}"))?;
+    let ds = file.dataset(dset).map_err(|e| format!("dataset: {e}"))?;
     ds.dtype().map_err(|e| format!("dtype: {e}"))?;
     ds.shape().map_err(|e| format!("shape: {e}"))?;
-    file.read_multi(&["d"])
+    file.read_multi(&[dset])
         .map(|_| ())
         .map_err(|e| format!("read: {e}"))
 }
@@ -93,10 +94,15 @@ fn clawhdf5_reads(path: &Path) -> Result<(), String> {
 /// h5py's verdict for each file must be `expected`, and clawhdf5 must read
 /// exactly the files h5py reads.
 fn assert_agrees_with_h5py(dir: &Path, verdicts: &[String], expected: &[&str]) {
+    assert_agrees_with_h5py_on(dir, verdicts, expected, "d");
+}
+
+/// [`assert_agrees_with_h5py`] reading dataset `dset`.
+fn assert_agrees_with_h5py_on(dir: &Path, verdicts: &[String], expected: &[&str], dset: &str) {
     assert_eq!(verdicts, expected, "h5py's view changed");
     for line in verdicts {
         let (name, verdict) = line.split_once(' ').unwrap();
-        let ours = clawhdf5_reads(&dir.join(format!("{name}.h5")));
+        let ours = clawhdf5_reads(&dir.join(format!("{name}.h5")), dset);
         match verdict {
             "ok" => assert!(ours.is_ok(), "{name}: h5py reads it, we fail: {ours:?}"),
             _ => assert!(ours.is_err(), "{name}: h5py refuses it, we read it"),
@@ -244,7 +250,7 @@ for libver in ("earliest", "latest"):
         ],
     );
     for name in ["earliest_size2", "latest_size8"] {
-        let err = clawhdf5_reads(&dir.path().join(format!("{name}.h5"))).unwrap_err();
+        let err = clawhdf5_reads(&dir.path().join(format!("{name}.h5")), "d").unwrap_err();
         assert!(
             err.contains("stored datatype size in chunk layout"),
             "{name}: {err}"
@@ -557,3 +563,68 @@ for name, n in (("overflow", (1 << 62) + 2), ("pasteof", 1000)):
         assert!(mm.dataset("d").is_err(), "{name}: MmapFile");
     }
 }
+
+/// libhdf5 decodes the superblock extension's messages when it opens a file
+/// and refuses the file when one does not decode: cve-2020-10810 (a File
+/// Space Info message too short for what it announces), cve-2020-10812 (a
+/// metadata cache image past the end of the file). We did not look at those
+/// messages and opened such files.
+#[test]
+fn superblock_extension_messages_libhdf5_refuses_are_refused() {
+    skip_if_no_python!();
+    let dir = tempfile::tempdir().unwrap();
+    let mdc = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/h5clear_mdc_image.h5");
+    let body = format!(
+        r#"
+{FIX_OHDR_PY}
+def ext_addr(buf):
+    assert buf[8] in (2, 3)
+    return int.from_bytes(buf[20:28], "little")
+def save(name, buf):
+    open(os.path.join(d, name + ".h5"), "wb").write(buf)
+
+# A paged file: its superblock extension holds a File Space Info message
+# (version 1, strategy page, not persisting, threshold 1, page size 4096).
+DSET = "DSET"
+good = os.path.join(d, "paged_good.h5")
+with h5py.File(good, "w", libver="latest", fs_strategy="page", fs_page_size=4096) as f:
+    f.create_dataset("DSET", data=np.arange(10, dtype="<i4"))
+data = bytearray(open(good, "rb").read())
+ext = ext_addr(data)
+at = data.find(struct.pack("<QQ", 1, 4096), ext)
+assert at > ext
+# Page size 256 (under libhdf5's minimum of 512).
+bad = bytearray(data); bad[at + 8:at + 16] = struct.pack("<Q", 256); fix_ohdr(bad, ext)
+save("paged_small_page", bad)
+# Persisting free space, without the manager addresses that then follow.
+bad = bytearray(data); bad[at - 1] = 1; fix_ohdr(bad, ext)
+save("paged_persist_short", bad)
+
+# h5clear_mdc_image.h5 (from libhdf5's tests): a metadata cache image, the
+# root group's header only in the image. The Metadata Cache Image message
+# (type 0x18) records the image's address and length.
+data = bytearray(open(r"{mdc}", "rb").read())
+save("mdc_good", data)
+ext = ext_addr(data)
+at = data.find(bytes([0x18, 17, 0]), ext)
+assert at > ext
+length = at + 5 + 8
+bad = bytearray(data); bad[length:length + 8] = struct.pack("<Q", 1 << 28); fix_ohdr(bad, ext)
+save("mdc_past_eof", bad)
+"#,
+        mdc = mdc.display()
+    );
+    let verdicts = h5py_verdicts(dir.path(), &body);
+    assert_agrees_with_h5py_on(
+        dir.path(),
+        &verdicts,
+        &[
+            "mdc_good ok",
+            "mdc_past_eof ERROR",
+            "paged_good ok",
+            "paged_persist_short ERROR",
+            "paged_small_page ERROR",
+        ],
+        "DSET",
+    );
+}

crates/clawhdf5/tests/metadata_cache_image.rs

@@ -0,0 +1,48 @@
+//! Files with a metadata cache image read as libhdf5 reads them.
+//!
+//! `fixtures/h5clear_mdc_image.h5` comes from libhdf5's tool tests
+//! (`tools/test/testfiles`): written with a metadata cache image, so its
+//! superblock extension points at an image block holding the file's
+//! metadata cache entries, and the root group's object header exists only
+//! there — the header's own address in the file is zeros. It holds one
+//! dataset, `/DSET`, 50 x 100 `int32` with `DSET[i][j] = i * j` (h5py
+//! 3.16 / HDF5 2.0 reads that). Until 2026-09-26 every reader failed on the
+//! root group: `InvalidObjectHeaderVersion(0)`.
+
+use std::path::PathBuf;
+
+use clawhdf5::{File, LazyFile, MmapFile};
+
+fn fixture() -> PathBuf {
+    PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/h5clear_mdc_image.h5")
+}
+
+fn expected() -> Vec<i32> {
+    (0..50).flat_map(|i| (0..100).map(move |j| i * j)).collect()
+}
+
+#[test]
+fn file_reads_through_the_cache_image() {
+    let file = File::open(fixture()).unwrap();
+    assert_eq!(file.root().datasets().unwrap(), ["DSET"]);
+    let ds = file.dataset("DSET").unwrap();
+    assert_eq!(ds.shape().unwrap(), [50, 100]);
+    assert_eq!(ds.read_i32().unwrap(), expected());
+
+    let file = File::from_bytes(std::fs::read(fixture()).unwrap()).unwrap();
+    assert_eq!(
+        file.dataset("DSET").unwrap().read_i32().unwrap(),
+        expected()
+    );
+}
+
+#[test]
+fn mmap_and_lazy_files_read_through_the_cache_image() {
+    let mm = MmapFile::open(fixture()).unwrap();
+    assert_eq!(mm.dataset("DSET").unwrap().read_i32().unwrap(), expected());
+    let lazy = LazyFile::from_bytes(std::fs::read(fixture()).unwrap()).unwrap();
+    assert_eq!(
+        lazy.dataset("DSET").unwrap().read_i32().unwrap(),
+        expected()
+    );
+}