format: a SWMR-flagged superblock's data ends at the end of the file

A SWMR writer does not keep the superblock's end-of-file address up to
date: a copy h5py made of its own file mid-write records 715 in a
6 030-byte file. Superblock::data_end only ignored the recorded end when
it lay past the end of the file, so every open path bounded reads at
715: the file listed, but every chunked read failed ("unexpected EOF:
need 787 bytes, have 715") and h5rs check reported the chunk indexes
past the end of the file. libhdf5's SWMR reader skips the
end-of-allocation check for every read (H5FD_read); for a v3 superblock
with the SWMR-write flag the data now ends at the end of the file.

Test: tests/swmr_interop.rs over the mid-write copy (fixture), through
File::open, open_buffered and from_bytes, and against h5py's SWMR reader.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>

crates/clawhdf5/tests/swmr_interop.rs

@@ -0,0 +1,107 @@
+//! Files a libhdf5 SWMR writer (h5py `f.swmr_mode = True`) has open: a copy
+//! taken mid-write (fixture), and a live file appended to by an h5py writer
+//! process while clawhdf5 and h5py's own SWMR reader read it (see
+//! `docs/design/swmr.md`).
+//!
+//! The live tests need python3 with h5py; they are skipped without it,
+//! unless `CLAWHDF5_REQUIRE_INTEROP=1`.
+
+use std::path::{Path, PathBuf};
+use std::process::Command;
+
+use clawhdf5::File;
+
+fn python() -> String {
+    std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
+}
+
+fn interop_required() -> bool {
+    std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
+}
+
+fn python_available() -> bool {
+    Command::new(python())
+        .args(["-c", "import h5py, numpy"])
+        .output()
+        .map(|o| o.status.success())
+        .unwrap_or(false)
+}
+
+macro_rules! skip_if_no_python {
+    () => {
+        if !python_available() {
+            assert!(
+                !interop_required(),
+                "CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available"
+            );
+            eprintln!("SKIP: python3 with h5py not available");
+            return;
+        }
+    };
+}
+
+fn fixture(name: &str) -> PathBuf {
+    Path::new(env!("CARGO_MANIFEST_DIR"))
+        .join("tests/fixtures")
+        .join(name)
+}
+
+/// `swmr_mid_write.h5`: a copy h5py 3.16 (HDF5 2.0) made of its own file
+/// while writing it in SWMR mode, after 4 appends of 37 rows to
+/// `/a` (int64, chunks of 100, no filter: `a[i] = i + 1`) and `/b` (float64
+/// `(n, 4)`, chunks of 16 x 4, gzip: `b[i, j] = 10 i + j + 1`), each
+/// followed by a flush. Its superblock (v3) still has the SWMR-write flag
+/// set and records an end of file of 715 in a 6 030-byte file.
+fn check_mid_write_copy(f: &File) {
+    let sb = f.superblock();
+    assert_eq!(sb.version, 3);
+    assert!(sb.is_swmr_write());
+    let a = f.dataset("a").unwrap();
+    assert_eq!(a.shape().unwrap(), vec![148]);
+    let want_a: Vec<i64> = (1..=148).collect();
+    assert_eq!(a.read_i64().unwrap(), want_a);
+    let b = f.dataset("b").unwrap();
+    assert_eq!(b.shape().unwrap(), vec![148, 4]);
+    let want_b: Vec<f64> = (0..148)
+        .flat_map(|i| (0..4).map(move |j| (10 * i + j + 1) as f64))
+        .collect();
+    assert_eq!(b.read_f64().unwrap(), want_b);
+}
+
+#[test]
+fn a_copy_taken_mid_write_reads_past_its_recorded_end_of_file() {
+    let path = fixture("swmr_mid_write.h5");
+    // The recorded end of file (715) is far below the file's length; the
+    // chunk indexes and chunks lie past it. libhdf5's SWMR reader does not
+    // bound reads by it, and neither does any open path here.
+    check_mid_write_copy(&File::open(&path).unwrap());
+    check_mid_write_copy(&File::open_buffered(&path).unwrap());
+    check_mid_write_copy(&File::from_bytes(std::fs::read(&path).unwrap()).unwrap());
+}
+
+#[test]
+fn a_copy_taken_mid_write_reads_as_h5py_swmr_reader_reads_it() {
+    skip_if_no_python!();
+    let path = fixture("swmr_mid_write.h5");
+    // libhdf5 refuses a non-SWMR open of this file ("file is already open
+    // for write"); its SWMR reader reads the values checked above.
+    let script = format!(
+        r#"
+import h5py, numpy as np
+with h5py.File("{p}", "r", swmr=True, locking=False) as f:
+    a = f["a"][()]
+    b = f["b"][()]
+assert np.array_equal(a, np.arange(148) + 1), a
+assert np.array_equal(b, (np.arange(148)[:, None] * 10 + np.arange(4) + 1).astype("f8")), b
+print("ok")
+"#,
+        p = path.display()
+    );
+    let out = Command::new(python()).args(["-c", &script]).output().unwrap();
+    assert!(
+        out.status.success(),
+        "h5py failed:\n{}",
+        String::from_utf8_lossy(&out.stderr)
+    );
+    check_mid_write_copy(&File::open(&path).unwrap());
+}