format: a SWMR-flagged superblock's data ends at the end of the file

A SWMR writer does not keep the superblock's end-of-file address up to
date: a copy h5py made of its own file mid-write records 715 in a
6 030-byte file. Superblock::data_end only ignored the recorded end when
it lay past the end of the file, so every open path bounded reads at
715: the file listed, but every chunked read failed ("unexpected EOF:
need 787 bytes, have 715") and h5rs check reported the chunk indexes
past the end of the file. libhdf5's SWMR reader skips the
end-of-allocation check for every read (H5FD_read); for a v3 superblock
with the SWMR-write flag the data now ends at the end of the file.

Test: tests/swmr_interop.rs over the mid-write copy (fixture), through
File::open, open_buffered and from_bytes, and against h5py's SWMR reader.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>

crates/clawhdf5-format/src/superblock.rs

@@ -116,22 +116,27 @@ impl Superblock {
     /// [`FormatError::TruncatedFile`]. Bytes past that address are not part
     /// of the file: libhdf5 fails any read of them ("addr overflow" /
     /// "address plus size exceeds file eoa"), so a reader should parse only
-    /// the data up to the returned end. As libhdf5 does for a SWMR reader,
+    /// the data up to the returned end.
-    /// the check is skipped for a version-3 superblock whose writer is still
+    ///
-    /// writing it in SWMR mode (it extends the file as it goes); the data
+    /// A version-3 superblock with the SWMR-write flag set belongs to a file
-    /// then ends at the end of the file.
+    /// a SWMR writer has open (or had, and did not close). That writer does
+    /// not keep the recorded end of file up to date — a copy taken mid-write
+    /// can record an end of a few hundred bytes in a file of tens of
+    /// kilobytes — and libhdf5's SWMR reader skips its end-of-allocation
+    /// check for every read (`H5FD_read`). For such a superblock the data
+    /// ends at the end of the file, whatever end it records.
     ///
     /// When the superblock's recorded base address differs from where the
     /// superblock actually is (a user block added or removed after the file
     /// was written), libhdf5 moves the recorded end of file by the same
     /// amount, and so does this.
     pub fn data_end(&self, user_block: u64, file_len: u64) -> Result<u64, FormatError> {
+        if self.version >= 3 && self.is_swmr_write() {
+            return Ok(file_len.saturating_sub(user_block));
+        }
         let eof =
             i128::from(self.eof_address) - i128::from(self.base_address) + i128::from(user_block);
         if eof < 0 || eof > i128::from(file_len) {
-            if self.version >= 3 && self.is_swmr_write() {
-                return Ok(file_len.saturating_sub(user_block));
-            }
             return Err(FormatError::TruncatedFile {
                 stored_eof: u64::try_from(eof).unwrap_or(self.eof_address),
                 actual_len: file_len,
@@ -617,6 +622,13 @@ mod tests {
         let mut swmr = Superblock::parse(&build_v2_bytes(8, 3), 0).unwrap();
         swmr.consistency_flags = swmr_flags::WRITE_ACCESS | swmr_flags::SWMR_WRITE;
         assert_eq!(swmr.data_end(0, 1000), Ok(1000));
+        // ... nor bounded by its recorded end, which the writer does not
+        // keep up to date (2048 here).
+        assert_eq!(swmr.data_end(0, 17_857), Ok(17_857));
+        assert_eq!(swmr.data_end(512, 17_857), Ok(17_345));
+        // Without the SWMR-write flag the recorded end bounds the data.
+        swmr.consistency_flags = swmr_flags::WRITE_ACCESS;
+        assert_eq!(swmr.data_end(0, 17_857), Ok(2048));
     }
 
     #[test]

crates/clawhdf5/tests/swmr_interop.rs

@@ -0,0 +1,107 @@
+//! Files a libhdf5 SWMR writer (h5py `f.swmr_mode = True`) has open: a copy
+//! taken mid-write (fixture), and a live file appended to by an h5py writer
+//! process while clawhdf5 and h5py's own SWMR reader read it (see
+//! `docs/design/swmr.md`).
+//!
+//! The live tests need python3 with h5py; they are skipped without it,
+//! unless `CLAWHDF5_REQUIRE_INTEROP=1`.
+
+use std::path::{Path, PathBuf};
+use std::process::Command;
+
+use clawhdf5::File;
+
+fn python() -> String {
+    std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
+}
+
+fn interop_required() -> bool {
+    std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
+}
+
+fn python_available() -> bool {
+    Command::new(python())
+        .args(["-c", "import h5py, numpy"])
+        .output()
+        .map(|o| o.status.success())
+        .unwrap_or(false)
+}
+
+macro_rules! skip_if_no_python {
+    () => {
+        if !python_available() {
+            assert!(
+                !interop_required(),
+                "CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available"
+            );
+            eprintln!("SKIP: python3 with h5py not available");
+            return;
+        }
+    };
+}
+
+fn fixture(name: &str) -> PathBuf {
+    Path::new(env!("CARGO_MANIFEST_DIR"))
+        .join("tests/fixtures")
+        .join(name)
+}
+
+/// `swmr_mid_write.h5`: a copy h5py 3.16 (HDF5 2.0) made of its own file
+/// while writing it in SWMR mode, after 4 appends of 37 rows to
+/// `/a` (int64, chunks of 100, no filter: `a[i] = i + 1`) and `/b` (float64
+/// `(n, 4)`, chunks of 16 x 4, gzip: `b[i, j] = 10 i + j + 1`), each
+/// followed by a flush. Its superblock (v3) still has the SWMR-write flag
+/// set and records an end of file of 715 in a 6 030-byte file.
+fn check_mid_write_copy(f: &File) {
+    let sb = f.superblock();
+    assert_eq!(sb.version, 3);
+    assert!(sb.is_swmr_write());
+    let a = f.dataset("a").unwrap();
+    assert_eq!(a.shape().unwrap(), vec![148]);
+    let want_a: Vec<i64> = (1..=148).collect();
+    assert_eq!(a.read_i64().unwrap(), want_a);
+    let b = f.dataset("b").unwrap();
+    assert_eq!(b.shape().unwrap(), vec![148, 4]);
+    let want_b: Vec<f64> = (0..148)
+        .flat_map(|i| (0..4).map(move |j| (10 * i + j + 1) as f64))
+        .collect();
+    assert_eq!(b.read_f64().unwrap(), want_b);
+}
+
+#[test]
+fn a_copy_taken_mid_write_reads_past_its_recorded_end_of_file() {
+    let path = fixture("swmr_mid_write.h5");
+    // The recorded end of file (715) is far below the file's length; the
+    // chunk indexes and chunks lie past it. libhdf5's SWMR reader does not
+    // bound reads by it, and neither does any open path here.
+    check_mid_write_copy(&File::open(&path).unwrap());
+    check_mid_write_copy(&File::open_buffered(&path).unwrap());
+    check_mid_write_copy(&File::from_bytes(std::fs::read(&path).unwrap()).unwrap());
+}
+
+#[test]
+fn a_copy_taken_mid_write_reads_as_h5py_swmr_reader_reads_it() {
+    skip_if_no_python!();
+    let path = fixture("swmr_mid_write.h5");
+    // libhdf5 refuses a non-SWMR open of this file ("file is already open
+    // for write"); its SWMR reader reads the values checked above.
+    let script = format!(
+        r#"
+import h5py, numpy as np
+with h5py.File("{p}", "r", swmr=True, locking=False) as f:
+    a = f["a"][()]
+    b = f["b"][()]
+assert np.array_equal(a, np.arange(148) + 1), a
+assert np.array_equal(b, (np.arange(148)[:, None] * 10 + np.arange(4) + 1).astype("f8")), b
+print("ok")
+"#,
+        p = path.display()
+    );
+    let out = Command::new(python()).args(["-c", &script]).output().unwrap();
+    assert!(
+        out.status.success(),
+        "h5py failed:\n{}",
+        String::from_utf8_lossy(&out.stderr)
+    );
+    check_mid_write_copy(&File::open(&path).unwrap());
+}