fix: apply a metadata cache image without copying the file

apply_cache_image returned a copy of the whole file with the image's
entries written in, and File (mmap by default), MmapFile and LazyFile
used that copy for every read: opening a 1 GiB sparse file with an image
needed 2 GB of memory, and an 8 GiB one aborted the process, where
de2a53f (which ignored the image) opened them in a few MB.

The metadata parsers read one contiguous slice, so the image still has
to be laid over the file's bytes; it is now laid over a private copy
that costs only the pages it touches:

- clawhdf5_format::superblock_ext::CacheImage decodes the image into an
  entry list (address, offset in the block, length) and applies it to
  any destination; cache_image_state tells an opener whether the file
  has no image, a loadable one, or one libhdf5 cannot load;
  apply_cache_image_in_place is for readers that own their buffer.
  apply_cache_image and metadata_view (which copied) are gone.
- clawhdf5_io::HDF5Read::private_copy returns a writable private copy
  of a reader's bytes: MmapReader gives a MAP_PRIVATE copy-on-write
  mapping (memmap2 map_copy), so only the pages the entries land on are
  copied; the default copies the bytes (in-memory readers).
- File, MmapFile and LazyFile write the image into that mapping
  (crate::cache_image). File::from_bytes / open_buffered patch their own
  buffer in place, copying only the image block, as libhdf5 does. A
  file without an image is read straight from the mapping, unchanged.

An image entry that runs past the end of file is now refused: libhdf5
checks only that it starts inside the file, and the images libhdf5
writes never do this, but those bytes have nowhere to go in a view of
the file.

Tests: tests/cache_image_memory.rs has libhdf5 (through ctypes) add an
image to a 1 GiB sparse file and bounds resident-memory growth for all
three openers at 256 MiB; it fails on the previous commit (File::open
grew 2,148,720,640 bytes). reader.rs zero_copy_tests check that a file
without an image is read from the mapping itself and that an image goes
into a copy-on-write mapping, not a heap copy; clawhdf5-io checks that
private_copy writes never reach the reader or the file.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>

crates/clawhdf5/src/cache_image.rs

@@ -0,0 +1,85 @@
+//! Metadata cache images, as the file openers apply them.
+//!
+//! A file written with a metadata cache image keeps metadata cache entries
+//! (object headers, B-tree nodes, heaps) in an image block, and libhdf5
+//! reads those entries in place of the file's own bytes at their addresses
+//! (see `clawhdf5_format::superblock_ext`). The metadata parsers read one
+//! contiguous byte slice, so the image has to be laid over the file's bytes
+//! — without copying the file:
+//!
+//! - an opener that holds the file in a buffer it owns writes the entries
+//!   into that buffer (only the image block is copied, as libhdf5 copies
+//!   it);
+//! - an opener that maps the file ([`File::open`](crate::File::open),
+//!   [`MmapFile`](crate::MmapFile), [`LazyFile::open_mmap`]
+//!   (crate::LazyFile::open_mmap)) writes them into a private copy-on-write
+//!   mapping of the file ([`clawhdf5_io::HDF5Read::private_copy`]): only the
+//!   pages the entries land on are copied, and the rest of the file stays
+//!   shared with the page cache;
+//! - a file without an image is read from the original bytes, as before.
+
+use clawhdf5_format::error::FormatError;
+use clawhdf5_format::superblock::Superblock;
+use clawhdf5_format::superblock_ext::{self, CacheImageState};
+use clawhdf5_io::PrivateCopy;
+
+use crate::error::Error;
+
+/// A file's metadata, as an opener must read it.
+pub(crate) enum ImageView {
+    /// Read the opener's bytes: the file has no cache image, or it was
+    /// written into a buffer the opener owns.
+    Plain,
+    /// The file with its cache image written in: a private copy of the
+    /// whole file (the HDF5 data at the same offsets as in the file).
+    Patched(PrivateCopy),
+    /// The file has a cache image libhdf5 cannot load.
+    Unloadable(FormatError),
+}
+
+/// Check the superblock extension of the file whose bytes are `whole` (the
+/// HDF5 data in `base..end`) and lay any cache image over a private copy
+/// of the file made by `copy`. An error means libhdf5 refuses the file.
+pub(crate) fn private_view(
+    whole: &[u8],
+    base: usize,
+    end: usize,
+    sb: &Superblock,
+    copy: impl FnOnce() -> std::io::Result<PrivateCopy>,
+) -> Result<ImageView, Error> {
+    let data = &whole[base..end];
+    Ok(match superblock_ext::cache_image_state(data, sb)? {
+        CacheImageState::Absent => ImageView::Plain,
+        CacheImageState::Unloadable(e) => ImageView::Unloadable(e),
+        CacheImageState::Loaded(image) => {
+            let mut view = copy().map_err(Error::Io)?;
+            let dst =
+                view.get_mut(base..end)
+                    .ok_or(Error::Format(FormatError::InvalidCacheImage(
+                        "the file changed while it was opened",
+                    )))?;
+            image.apply(image.block(data)?, dst)?;
+            ImageView::Patched(view)
+        }
+    })
+}
+
+/// [`private_view`] for a file held in `whole`, a buffer the opener owns:
+/// the image is written into it in place.
+pub(crate) fn in_place(
+    whole: &mut [u8],
+    base: usize,
+    end: usize,
+    sb: &Superblock,
+) -> Result<ImageView, Error> {
+    let data = &mut whole[base..end];
+    Ok(match superblock_ext::cache_image_state(data, sb)? {
+        CacheImageState::Absent => ImageView::Plain,
+        CacheImageState::Unloadable(e) => ImageView::Unloadable(e),
+        CacheImageState::Loaded(image) => {
+            let block = image.block(data)?.to_vec();
+            image.apply(&block, data)?;
+            ImageView::Plain
+        }
+    })
+}

crates/clawhdf5/src/lazy.rs

@@ -46,9 +46,13 @@ pub struct LazyFile<R: HDF5Read> {
     /// End of the HDF5 data (`Superblock::data_end`, absolute).
     end: usize,
     superblock: Superblock,
-    /// The metadata as libhdf5 reads it when the file holds a metadata
-    /// cache image (see `superblock_ext::metadata_view`); `None` otherwise.
-    overlay: Option<Vec<u8>>,
+    /// A file that holds a metadata cache image, with the image written in:
+    /// the reader's [`HDF5Read::private_copy`] of the whole file (a
+    /// copy-on-write mapping for [`clawhdf5_io::MmapReader`], so only the
+    /// pages the image's entries land on are copied; see
+    /// `crate::cache_image`). `None` for a file without an image, read
+    /// straight from the reader.
+    patched: Option<clawhdf5_io::PrivateCopy>,
     root_header: ObjectHeader,
     /// Cache of parsed object headers, keyed by address.
     header_cache: RefCell<HashMap<u64, ObjectHeader>>,
@@ -87,11 +91,19 @@ impl<R: HDF5Read> LazyFile<R> {
         let end = base + superblock.data_end(base as u64, whole_len)? as usize;
         // Decode the superblock extension as libhdf5 does at open, and load
         // a metadata cache image over the file's metadata.
-        let overlay = clawhdf5_format::superblock_ext::metadata_view(
-            &reader.as_bytes()[base..end],
-            &superblock,
-        )?;
-        let data = overlay.as_deref().unwrap_or(&reader.as_bytes()[base..end]);
+        let view =
+            crate::cache_image::private_view(reader.as_bytes(), base, end, &superblock, || {
+                reader.private_copy()
+            })?;
+        let patched = match view {
+            crate::cache_image::ImageView::Plain => None,
+            crate::cache_image::ImageView::Patched(p) => Some(p),
+            crate::cache_image::ImageView::Unloadable(e) => return Err(e.into()),
+        };
+        let data = match &patched {
+            Some(p) => &p[base..end],
+            None => &reader.as_bytes()[base..end],
+        };
         let root_header = ObjectHeader::parse(
             data,
             superblock.root_group_address as usize,
@@ -103,7 +115,7 @@ impl<R: HDF5Read> LazyFile<R> {
             base,
             end,
             superblock,
-            overlay,
+            patched,
             root_header,
             header_cache: RefCell::new(HashMap::new()),
         })
@@ -121,8 +133,8 @@ impl<R: HDF5Read> LazyFile<R> {
     }
 
     fn hdf5_bytes(&self) -> &[u8] {
-        match &self.overlay {
-            Some(v) => v,
+        match &self.patched {
+            Some(p) => &p[self.base..self.end],
             None => &self.reader.as_bytes()[self.base..self.end],
         }
     }

crates/clawhdf5/src/lib.rs

@@ -24,6 +24,7 @@
 //! builder.write("output.h5").unwrap();
 //! ```
 
+mod cache_image;
 pub mod error;
 pub mod lazy;
 #[cfg(feature = "mmap")]

crates/clawhdf5/src/mmap_file.rs

@@ -38,9 +38,11 @@ pub struct MmapFile {
     /// End of the HDF5 data (`Superblock::data_end`, absolute).
     end: usize,
     superblock: Superblock,
-    /// The metadata as libhdf5 reads it when the file holds a metadata
-    /// cache image (see `superblock_ext::metadata_view`); `None` otherwise.
-    overlay: Option<Vec<u8>>,
+    /// A file that holds a metadata cache image, with the image written in:
+    /// a private copy-on-write mapping of the whole file, so only the pages
+    /// the image's entries land on are copied (see `crate::cache_image`).
+    /// `None` for a file without an image, read straight from the mapping.
+    patched: Option<clawhdf5_io::PrivateCopy>,
 }
 
 impl MmapFile {
@@ -55,24 +57,29 @@ impl MmapFile {
         let end = base + superblock.data_end(base as u64, whole_len)? as usize;
         // Decode the superblock extension as libhdf5 does at open, and load
         // a metadata cache image over the file's metadata.
-        let overlay = clawhdf5_format::superblock_ext::metadata_view(
-            &reader.as_bytes()[base..end],
-            &superblock,
-        )?;
+        let view =
+            crate::cache_image::private_view(reader.as_bytes(), base, end, &superblock, || {
+                clawhdf5_io::HDF5Read::private_copy(&reader)
+            })?;
+        let patched = match view {
+            crate::cache_image::ImageView::Plain => None,
+            crate::cache_image::ImageView::Patched(p) => Some(p),
+            crate::cache_image::ImageView::Unloadable(e) => return Err(e.into()),
+        };
         Ok(Self {
             reader,
             base,
             end,
             superblock,
-            overlay,
+            patched,
         })
     }
 
     /// The file's bytes from the superblock on — the space HDF5 addresses
     /// index into.
     fn hdf5_bytes(&self) -> &[u8] {
-        match &self.overlay {
-            Some(v) => v,
+        match &self.patched {
+            Some(p) => &p[self.base..self.end],
             None => &self.reader.as_bytes()[self.base..self.end],
         }
     }

crates/clawhdf5/src/reader.rs

@@ -20,8 +20,8 @@ use clawhdf5_format::message_type::MessageType;
 use clawhdf5_format::object_header::ObjectHeader;
 use clawhdf5_format::signature;
 use clawhdf5_format::superblock::Superblock;
-use clawhdf5_format::superblock_ext;
 
+use crate::cache_image::{self, ImageView};
 use crate::error::Error;
 use crate::types::{AttrValue, DType, classify_datatype, read_attrs};
 
@@ -56,17 +56,19 @@ struct FileData {
     base: usize,
     /// End of the HDF5 data in the file (`Superblock::data_end`, absolute).
     end: usize,
-    /// The file's metadata as libhdf5 reads it when the file holds a
-    /// metadata cache image: the bytes from the superblock to the end of
-    /// file with the image's entries written in
-    /// ([`superblock_ext::metadata_view`]). `None` for every other file.
-    overlay: Option<Vec<u8>>,
+    /// A mapped file that holds a metadata cache image, with the image
+    /// written in: a private copy-on-write mapping of the whole file, so
+    /// only the pages the image's entries land on are copied (see
+    /// `crate::cache_image`). `None` for every other file: a file without
+    /// an image is read straight from the mapping, and an owned buffer has
+    /// the image written into it in place.
+    patched: Option<clawhdf5_io::PrivateCopy>,
 }
 
 impl FileData {
     /// Locate the superblock and parse it. A truncated file is refused, and
     /// bytes past the recorded end of file are not read, as in libhdf5.
-    fn new(backing: Backing) -> Result<(Self, Superblock), Error> {
+    fn new(mut backing: Backing) -> Result<(Self, Superblock), Error> {
         let whole = backing.whole_file();
         let (user_block, hdf5) = signature::split_user_block(whole)?;
         let base = user_block.len();
@@ -77,21 +79,34 @@ impl FileData {
         // libhdf5 decodes the superblock extension at open (a message it
         // cannot decode fails the open) and loads a metadata cache image
         // over the file's own metadata.
-        let overlay = superblock_ext::metadata_view(&whole[base..end], &superblock)?;
+        let view = match &mut backing {
+            Backing::Owned(v) => cache_image::in_place(v, base, end, &superblock)?,
+            #[cfg(feature = "mmap")]
+            Backing::Mmap(r) => {
+                cache_image::private_view(r.as_bytes(), base, end, &superblock, || {
+                    clawhdf5_io::HDF5Read::private_copy(r)
+                })?
+            }
+        };
+        let patched = match view {
+            ImageView::Plain => None,
+            ImageView::Patched(p) => Some(p),
+            ImageView::Unloadable(e) => return Err(e.into()),
+        };
         Ok((
             Self {
                 backing,
                 base,
                 end,
-                overlay,
+                patched,
             },
             superblock,
         ))
     }
 
     fn as_bytes(&self) -> &[u8] {
-        match &self.overlay {
-            Some(v) => v,
+        match &self.patched {
+            Some(p) => &p[self.base..self.end],
             None => &self.backing.whole_file()[self.base..self.end],
         }
     }
@@ -1282,3 +1297,44 @@ mod sibling_file_name_tests {
         }
     }
 }
+
+#[cfg(all(test, feature = "mmap"))]
+mod zero_copy_tests {
+    use super::*;
+
+    /// Where `File::open` reads metadata from: `Some(true)` for the file's
+    /// own mapping, `Some(false)` for a private copy-on-write mapping.
+    fn reads_from_the_mapping(f: &File) -> Option<bool> {
+        let Backing::Mmap(r) = &f.data.backing else {
+            return None;
+        };
+        let mapped = r.as_bytes()[f.data.base..].as_ptr();
+        match &f.data.patched {
+            None => Some(std::ptr::eq(f.as_bytes().as_ptr(), mapped)),
+            Some(p) => {
+                assert!(p.is_mapped(), "the image went into a heap copy of the file");
+                Some(false)
+            }
+        }
+    }
+
+    #[test]
+    fn a_file_without_a_cache_image_is_read_from_the_mapping() {
+        let dir = tempfile::tempdir().unwrap();
+        let path = dir.path().join("plain.h5");
+        let mut b = crate::FileBuilder::new();
+        b.create_dataset("d").with_f64_data(&[1.0, 2.0]);
+        b.write(&path).unwrap();
+        let f = File::open(&path).unwrap();
+        assert_eq!(reads_from_the_mapping(&f), Some(true));
+        assert_eq!(f.dataset("d").unwrap().read_f64().unwrap(), [1.0, 2.0]);
+    }
+
+    #[test]
+    fn a_cache_image_goes_into_a_copy_on_write_mapping() {
+        let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
+            .join("tests/fixtures/h5clear_mdc_image.h5");
+        let f = File::open(path).unwrap();
+        assert_eq!(reads_from_the_mapping(&f), Some(false));
+    }
+}

crates/clawhdf5/tests/cache_image_memory.rs

@@ -0,0 +1,139 @@
+//! Opening a file with a metadata cache image must not copy the file.
+//!
+//! The image's entries are laid over the file's bytes in a private
+//! copy-on-write mapping (`File::open`, `MmapFile::open`,
+//! `LazyFile::open_mmap`), so only the pages they land on are copied. The
+//! first implementation copied the whole file onto the heap at open: a
+//! 1 GiB file that takes a few KB on disk needed 2 GB of memory, and an
+//! 8 GiB one aborted the process. Here libhdf5 itself (the library h5py
+//! bundles, through ctypes: `H5Pset_mdc_image_config`) adds an image to a
+//! 1 GiB sparse file, and the process's resident memory must stay far below
+//! the file's size while each opener lists the file and reads its small
+//! dataset.
+//!
+//! One test in its own binary, so no other test's allocations land in the
+//! measurement. Linux only (it reads `VmRSS` from `/proc/self/status`).
+//! Skipped when python3 with h5py is unavailable, unless
+//! `CLAWHDF5_REQUIRE_INTEROP=1`.
+
+#![cfg(target_os = "linux")]
+
+use std::path::Path;
+use std::process::Command;
+
+use clawhdf5::{File, LazyFile, MmapFile};
+
+fn python() -> String {
+    std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
+}
+
+fn python_available() -> bool {
+    Command::new(python())
+        .args(["-c", "import h5py, numpy"])
+        .output()
+        .map(|o| o.status.success())
+        .unwrap_or(false)
+}
+
+fn rss_bytes() -> u64 {
+    let status = std::fs::read_to_string("/proc/self/status").unwrap();
+    let line = status.lines().find(|l| l.starts_with("VmRSS:")).unwrap();
+    let kb: u64 = line.split_whitespace().nth(1).unwrap().parse().unwrap();
+    kb * 1024
+}
+
+/// A 1 GiB sparse file: `/big`, 2^27 `f8` with only its last element
+/// written, `/small` = 0..10, with a metadata cache image added by libhdf5.
+fn make_file(path: &Path) {
+    let script = format!(
+        r#"
+import ctypes, glob, os, h5py, numpy as np
+path = "{path}"
+libs = glob.glob(os.path.join(os.path.dirname(h5py.__file__), os.pardir, "h5py.libs", "libhdf5-*.so*"))
+assert libs, "no libhdf5 bundled with h5py"
+lib = ctypes.CDLL(libs[0])
+class Cfg(ctypes.Structure):
+    _fields_ = [("version", ctypes.c_int), ("generate_image", ctypes.c_bool),
+                ("save_resize_status", ctypes.c_bool), ("entry_ageout", ctypes.c_int)]
+with h5py.File(path, "w", libver="latest") as f:
+    d = f.create_dataset("big", shape=(2**27,), dtype="f8")
+    d[-1] = 7.5
+    f.create_dataset("small", data=np.arange(10, dtype="<i4"))
+fapl = h5py.h5p.create(h5py.h5p.FILE_ACCESS)
+fapl.set_libver_bounds(h5py.h5f.LIBVER_LATEST, h5py.h5f.LIBVER_LATEST)
+cfg = Cfg(1, True, False, -1)
+assert lib.H5Pset_mdc_image_config(ctypes.c_int64(fapl.id), ctypes.byref(cfg)) >= 0
+f = h5py.File(h5py.h5f.open(path.encode(), h5py.h5f.ACC_RDWR, fapl=fapl))
+f["small"][()]; f["big"].shape
+f.close()
+assert os.path.getsize(path) >= 2**30
+with open(path, "rb") as fh:
+    fh.seek(-(1 << 20), 2)
+    assert b"MDCI" in fh.read(), "libhdf5 wrote no cache image"
+with h5py.File(path, "r") as f:
+    assert list(f["small"][()]) == list(range(10))
+"#,
+        path = path.display()
+    );
+    let out = Command::new(python())
+        .args(["-c", &script])
+        .output()
+        .expect("failed to run python");
+    assert!(
+        out.status.success(),
+        "python failed:\n{}",
+        String::from_utf8_lossy(&out.stderr)
+    );
+}
+
+#[test]
+fn a_cache_image_does_not_copy_the_file() {
+    if !python_available() {
+        assert!(
+            !std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1"),
+            "CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available"
+        );
+        eprintln!("SKIP: python3 with h5py not available");
+        return;
+    }
+    let dir = tempfile::tempdir().unwrap();
+    let path = dir.path().join("sparse_image.h5");
+    make_file(&path);
+
+    const LIMIT: u64 = 256 << 20;
+    let small: Vec<i32> = (0..10).collect();
+    let before = rss_bytes();
+    {
+        let f = File::open(&path).unwrap();
+        let mut names = f.root().datasets().unwrap();
+        names.sort();
+        assert_eq!(names, ["big", "small"]);
+        assert_eq!(f.dataset("small").unwrap().read_i32().unwrap(), small);
+        assert_eq!(f.dataset("big").unwrap().shape().unwrap(), [1 << 27]);
+        let grew = rss_bytes().saturating_sub(before);
+        assert!(
+            grew < LIMIT,
+            "File::open: resident memory grew {grew} bytes"
+        );
+    }
+    let before = rss_bytes();
+    {
+        let f = MmapFile::open(&path).unwrap();
+        assert_eq!(f.dataset("small").unwrap().read_i32().unwrap(), small);
+        let grew = rss_bytes().saturating_sub(before);
+        assert!(
+            grew < LIMIT,
+            "MmapFile::open: resident memory grew {grew} bytes"
+        );
+    }
+    let before = rss_bytes();
+    {
+        let f = LazyFile::open_mmap(&path).unwrap();
+        assert_eq!(f.dataset("small").unwrap().read_i32().unwrap(), small);
+        let grew = rss_bytes().saturating_sub(before);
+        assert!(
+            grew < LIMIT,
+            "LazyFile::open_mmap: resident memory grew {grew} bytes"
+        );
+    }
+}