From e39737682031c49fbbb90719dc17b70531f8e5da Mon Sep 17 00:00:00 2001 From: osobh Date: Sun, 27 Sep 2026 06:31:15 -0500 Subject: [PATCH] format: a SWMR-flagged superblock's data ends at the end of the file A SWMR writer does not keep the superblock's end-of-file address up to date: a copy h5py made of its own file mid-write records 715 in a 6 030-byte file. Superblock::data_end only ignored the recorded end when it lay past the end of the file, so every open path bounded reads at 715: the file listed, but every chunked read failed ("unexpected EOF: need 787 bytes, have 715") and h5rs check reported the chunk indexes past the end of the file. libhdf5's SWMR reader skips the end-of-allocation check for every read (H5FD_read); for a v3 superblock with the SWMR-write flag the data now ends at the end of the file. Test: tests/swmr_interop.rs over the mid-write copy (fixture), through File::open, open_buffered and from_bytes, and against h5py's SWMR reader. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/superblock.rs | 26 +++-- .../clawhdf5/tests/fixtures/swmr_mid_write.h5 | Bin 0 -> 6030 bytes crates/clawhdf5/tests/swmr_interop.rs | 107 ++++++++++++++++++ 3 files changed, 126 insertions(+), 7 deletions(-) create mode 100644 crates/clawhdf5/tests/fixtures/swmr_mid_write.h5 create mode 100644 crates/clawhdf5/tests/swmr_interop.rs diff --git a/crates/clawhdf5-format/src/superblock.rs b/crates/clawhdf5-format/src/superblock.rs index dc21f7c..5ae4daa 100644 --- a/crates/clawhdf5-format/src/superblock.rs +++ b/crates/clawhdf5-format/src/superblock.rs @@ -116,22 +116,27 @@ impl Superblock { /// [`FormatError::TruncatedFile`]. Bytes past that address are not part /// of the file: libhdf5 fails any read of them ("addr overflow" / /// "address plus size exceeds file eoa"), so a reader should parse only - /// the data up to the returned end. As libhdf5 does for a SWMR reader, - /// the check is skipped for a version-3 superblock whose writer is still - /// writing it in SWMR mode (it extends the file as it goes); the data - /// then ends at the end of the file. + /// the data up to the returned end. + /// + /// A version-3 superblock with the SWMR-write flag set belongs to a file + /// a SWMR writer has open (or had, and did not close). That writer does + /// not keep the recorded end of file up to date — a copy taken mid-write + /// can record an end of a few hundred bytes in a file of tens of + /// kilobytes — and libhdf5's SWMR reader skips its end-of-allocation + /// check for every read (`H5FD_read`). For such a superblock the data + /// ends at the end of the file, whatever end it records. /// /// When the superblock's recorded base address differs from where the /// superblock actually is (a user block added or removed after the file /// was written), libhdf5 moves the recorded end of file by the same /// amount, and so does this. pub fn data_end(&self, user_block: u64, file_len: u64) -> Result { + if self.version >= 3 && self.is_swmr_write() { + return Ok(file_len.saturating_sub(user_block)); + } let eof = i128::from(self.eof_address) - i128::from(self.base_address) + i128::from(user_block); if eof < 0 || eof > i128::from(file_len) { - if self.version >= 3 && self.is_swmr_write() { - return Ok(file_len.saturating_sub(user_block)); - } return Err(FormatError::TruncatedFile { stored_eof: u64::try_from(eof).unwrap_or(self.eof_address), actual_len: file_len, @@ -617,6 +622,13 @@ mod tests { let mut swmr = Superblock::parse(&build_v2_bytes(8, 3), 0).unwrap(); swmr.consistency_flags = swmr_flags::WRITE_ACCESS | swmr_flags::SWMR_WRITE; assert_eq!(swmr.data_end(0, 1000), Ok(1000)); + // ... nor bounded by its recorded end, which the writer does not + // keep up to date (2048 here). + assert_eq!(swmr.data_end(0, 17_857), Ok(17_857)); + assert_eq!(swmr.data_end(512, 17_857), Ok(17_345)); + // Without the SWMR-write flag the recorded end bounds the data. + swmr.consistency_flags = swmr_flags::WRITE_ACCESS; + assert_eq!(swmr.data_end(0, 17_857), Ok(2048)); } #[test] diff --git a/crates/clawhdf5/tests/fixtures/swmr_mid_write.h5 b/crates/clawhdf5/tests/fixtures/swmr_mid_write.h5 new file mode 100644 index 0000000000000000000000000000000000000000..28cbb9487345a2fa7d1d9653c61c48cab8d5d79b GIT binary patch literal 6030 zcmeI#c~nzZ8UXMY2qq8+VNoe-aJR^&D3rwziv&ekWE49{6mb-5WnTg@pg^EV1rZP( z2%sH{)2X7h6+{6cAQd;LC|FsH?6L|}7D0Mn@;wiK%sF$;{AEwQN8j(=`|f-1%a@yb zFG)@gjw(vpO7c{y0tRIk$~`$*gBi3hlE2Gzb#mAyhwJOhL@vniVTha%b$LyUq{d{u&y58Q zvzV2~AV)NmIrI?x+r#iXs0*Z5>!h#|ZKPKXzwp(rgvNOZtKjpU8&O&c(b7-sjNYXw8r zllW%->RKFi;z=__BF0vqiY!BEGNi{NbXg^7PMeFWtdcONtpVB+=d`uK#P(OqbeOMs z>PEK12JC%i@uWyJWT}~(AwL+tU(d-}G@38*Ld+=mxjLE;@RBb*$__t5Tkf1Tw}rCG z`Z;Yx_UAOcUqAj#ZNGS!p!Mxz`Iv!^8Tha=K&;Kg+D>d$h`kT7znuqRQ$X215yKRr zPORxPs1rYTSe7tYiZEDeFj&wrHRvzh8l!WVXfa}M023+(6D9_eB&G%TeFAEOI-oA7 z2kL_cAQLnM7lDhxCE!wU8Mqu=0j>l;1&zQ}pfP9ynu5fi2j);;4Xyz#z_s8y&=Ms6 z9QX|C)}Re&3)+Fis=|W0J;(;prb}8r2hpZbS~h}?AO~~;H-Vc$XK)L+6?6ezK{wDH z+y;7pe*?FJJ3voxC-?>UchC#m1$u*Aa5uOI^a1yRzTiHP$6Z^qiB59Ybd<#S`6)Us z+WY<4Cz(w4`Q&BC-5MMRx2@_lu2x^gF76iH&00J&r5Uq?rrlJvF8oTeU4J{4XjLg^ z9NKUuZ`ebBZF~20EXR0qetDOBw%OrHZ@-?&aoc4L$5a&z-sD6^6n2F_xM0tG24$t5E4BYSUY+tUg;x&D&CMCGT-Nk>y3D)SDt9N$uEWUc} zsc%zz|J&x~cObKes&!+uSHtNic8&bQ1cxG9b|(k{Cy84jM$xj#{nqEq{F!4Jjf zji(fzE#^pgT;DyI+%a_uqdFhEppyTSd`2sFLDjyO<~Bvjpsx_tDY}(ln=S2oRo(u1 zPI&{({-DANJJM#;qrd9pkC9f`Df!LA8hHyHcwFOZXEk|4hJ8iaQSuU|&k`m{QTJ5K z^mQdgCwn>dlX#=yrU!k_$Q(koG;gFoRlKq^XzZ!bE23g66ft}Q(7lWH~XI5fZ28HfX$%qz{55r$98B+?Ue zJe2()P^&a!Ocq zCkI;N%DK0qMh)JQa>G}z{0EHg{on!64?GBd2_6E!0uO`!U;r2h27$p~2p9^6f#F~T z$O9w6D3A{x0R><*7y}*!kAbmZ92gG@!2~c7{2ELGkAunJH{b~{1xy7`f@$C>FdfVQ zKk{K`0G|x>v?}c}tWJ#o?zHq1kcmEZk0gOs%3}PGC^q|{jPJ^&vXY&r7oO%u4wR~{ zFBuWLMhwmb6?ZM!-O6v^9x93Lw* zJ>hLWT06R1j4b2x`ZZ^T!mNj-K?9c}1_k3U5?S=_&@S}zZ!LcFK*VpedpnRSURoL+ z+FQ#hD!Vb-k#(Tld)H&5LcP-Q3QkdU)L2JW)>i8XNoG-NoR|k|01a%_p{$CoeOWh zsSu``2*NtiEIQGf)#g8xy*NUb-;-A3GF?Nd_u5>KXJt+_WHw_AW22zhX83<Wmim^=H9S6p1RU`#hlV*j#9FDL95i#Kofdqz}D)w9nFY)we+!{-e#8ec# z)yLo2I?@apYF&EYlPu6Co%v?0N8{c_e!XTYqLTB)mb z=T9{HjO6^G%Qq$8p(pewHMa8lMCb{F9L;wm14U0|JU+wIY!u&Z18H!Kc$HWtwNe#7 z%##JkZn{Fbt$|;q$O0xwd){8-v5eZfNIdMp2t(0`LRu9Xr~68D(09g~nD~qciP%rta@+JOsqp&R4iMx z6HV{>y8kHMO2nQ%NM{~DC%<{zzE{yLN(J9d6xsLcxlJi$l#wnB+4oZ1rc^WbpP`@o SBHx0Np|2o659{JAsQLp{_WNA` literal 0 HcmV?d00001 diff --git a/crates/clawhdf5/tests/swmr_interop.rs b/crates/clawhdf5/tests/swmr_interop.rs new file mode 100644 index 0000000..f22e7cd --- /dev/null +++ b/crates/clawhdf5/tests/swmr_interop.rs @@ -0,0 +1,107 @@ +//! Files a libhdf5 SWMR writer (h5py `f.swmr_mode = True`) has open: a copy +//! taken mid-write (fixture), and a live file appended to by an h5py writer +//! process while clawhdf5 and h5py's own SWMR reader read it (see +//! `docs/design/swmr.md`). +//! +//! The live tests need python3 with h5py; they are skipped without it, +//! unless `CLAWHDF5_REQUIRE_INTEROP=1`. + +use std::path::{Path, PathBuf}; +use std::process::Command; + +use clawhdf5::File; + +fn python() -> String { + std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) +} + +fn interop_required() -> bool { + std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1") +} + +fn python_available() -> bool { + Command::new(python()) + .args(["-c", "import h5py, numpy"]) + .output() + .map(|o| o.status.success()) + .unwrap_or(false) +} + +macro_rules! skip_if_no_python { + () => { + if !python_available() { + assert!( + !interop_required(), + "CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available" + ); + eprintln!("SKIP: python3 with h5py not available"); + return; + } + }; +} + +fn fixture(name: &str) -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures") + .join(name) +} + +/// `swmr_mid_write.h5`: a copy h5py 3.16 (HDF5 2.0) made of its own file +/// while writing it in SWMR mode, after 4 appends of 37 rows to +/// `/a` (int64, chunks of 100, no filter: `a[i] = i + 1`) and `/b` (float64 +/// `(n, 4)`, chunks of 16 x 4, gzip: `b[i, j] = 10 i + j + 1`), each +/// followed by a flush. Its superblock (v3) still has the SWMR-write flag +/// set and records an end of file of 715 in a 6 030-byte file. +fn check_mid_write_copy(f: &File) { + let sb = f.superblock(); + assert_eq!(sb.version, 3); + assert!(sb.is_swmr_write()); + let a = f.dataset("a").unwrap(); + assert_eq!(a.shape().unwrap(), vec![148]); + let want_a: Vec = (1..=148).collect(); + assert_eq!(a.read_i64().unwrap(), want_a); + let b = f.dataset("b").unwrap(); + assert_eq!(b.shape().unwrap(), vec![148, 4]); + let want_b: Vec = (0..148) + .flat_map(|i| (0..4).map(move |j| (10 * i + j + 1) as f64)) + .collect(); + assert_eq!(b.read_f64().unwrap(), want_b); +} + +#[test] +fn a_copy_taken_mid_write_reads_past_its_recorded_end_of_file() { + let path = fixture("swmr_mid_write.h5"); + // The recorded end of file (715) is far below the file's length; the + // chunk indexes and chunks lie past it. libhdf5's SWMR reader does not + // bound reads by it, and neither does any open path here. + check_mid_write_copy(&File::open(&path).unwrap()); + check_mid_write_copy(&File::open_buffered(&path).unwrap()); + check_mid_write_copy(&File::from_bytes(std::fs::read(&path).unwrap()).unwrap()); +} + +#[test] +fn a_copy_taken_mid_write_reads_as_h5py_swmr_reader_reads_it() { + skip_if_no_python!(); + let path = fixture("swmr_mid_write.h5"); + // libhdf5 refuses a non-SWMR open of this file ("file is already open + // for write"); its SWMR reader reads the values checked above. + let script = format!( + r#" +import h5py, numpy as np +with h5py.File("{p}", "r", swmr=True, locking=False) as f: + a = f["a"][()] + b = f["b"][()] +assert np.array_equal(a, np.arange(148) + 1), a +assert np.array_equal(b, (np.arange(148)[:, None] * 10 + np.arange(4) + 1).astype("f8")), b +print("ok") +"#, + p = path.display() + ); + let out = Command::new(python()).args(["-c", &script]).output().unwrap(); + assert!( + out.status.success(), + "h5py failed:\n{}", + String::from_utf8_lossy(&out.stderr) + ); + check_mid_write_copy(&File::open(&path).unwrap()); +}