format: a SWMR-flagged superblock's data ends at the end of the file
A SWMR writer does not keep the superblock's end-of-file address up to
date: a copy h5py made of its own file mid-write records 715 in a
6 030-byte file. Superblock::data_end only ignored the recorded end when
it lay past the end of the file, so every open path bounded reads at
715: the file listed, but every chunked read failed ("unexpected EOF:
need 787 bytes, have 715") and h5rs check reported the chunk indexes
past the end of the file. libhdf5's SWMR reader skips the
end-of-allocation check for every read (H5FD_read); for a v3 superblock
with the SWMR-write flag the data now ends at the end of the file.
Test: tests/swmr_interop.rs over the mid-write copy (fixture), through
File::open, open_buffered and from_bytes, and against h5py's SWMR reader.
Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
@@ -116,22 +116,27 @@ impl Superblock {
|
||||
/// [`FormatError::TruncatedFile`]. Bytes past that address are not part
|
||||
/// of the file: libhdf5 fails any read of them ("addr overflow" /
|
||||
/// "address plus size exceeds file eoa"), so a reader should parse only
|
||||
/// the data up to the returned end. As libhdf5 does for a SWMR reader,
|
||||
/// the check is skipped for a version-3 superblock whose writer is still
|
||||
/// writing it in SWMR mode (it extends the file as it goes); the data
|
||||
/// then ends at the end of the file.
|
||||
/// the data up to the returned end.
|
||||
///
|
||||
/// A version-3 superblock with the SWMR-write flag set belongs to a file
|
||||
/// a SWMR writer has open (or had, and did not close). That writer does
|
||||
/// not keep the recorded end of file up to date — a copy taken mid-write
|
||||
/// can record an end of a few hundred bytes in a file of tens of
|
||||
/// kilobytes — and libhdf5's SWMR reader skips its end-of-allocation
|
||||
/// check for every read (`H5FD_read`). For such a superblock the data
|
||||
/// ends at the end of the file, whatever end it records.
|
||||
///
|
||||
/// When the superblock's recorded base address differs from where the
|
||||
/// superblock actually is (a user block added or removed after the file
|
||||
/// was written), libhdf5 moves the recorded end of file by the same
|
||||
/// amount, and so does this.
|
||||
pub fn data_end(&self, user_block: u64, file_len: u64) -> Result<u64, FormatError> {
|
||||
if self.version >= 3 && self.is_swmr_write() {
|
||||
return Ok(file_len.saturating_sub(user_block));
|
||||
}
|
||||
let eof =
|
||||
i128::from(self.eof_address) - i128::from(self.base_address) + i128::from(user_block);
|
||||
if eof < 0 || eof > i128::from(file_len) {
|
||||
if self.version >= 3 && self.is_swmr_write() {
|
||||
return Ok(file_len.saturating_sub(user_block));
|
||||
}
|
||||
return Err(FormatError::TruncatedFile {
|
||||
stored_eof: u64::try_from(eof).unwrap_or(self.eof_address),
|
||||
actual_len: file_len,
|
||||
@@ -617,6 +622,13 @@ mod tests {
|
||||
let mut swmr = Superblock::parse(&build_v2_bytes(8, 3), 0).unwrap();
|
||||
swmr.consistency_flags = swmr_flags::WRITE_ACCESS | swmr_flags::SWMR_WRITE;
|
||||
assert_eq!(swmr.data_end(0, 1000), Ok(1000));
|
||||
// ... nor bounded by its recorded end, which the writer does not
|
||||
// keep up to date (2048 here).
|
||||
assert_eq!(swmr.data_end(0, 17_857), Ok(17_857));
|
||||
assert_eq!(swmr.data_end(512, 17_857), Ok(17_345));
|
||||
// Without the SWMR-write flag the recorded end bounds the data.
|
||||
swmr.consistency_flags = swmr_flags::WRITE_ACCESS;
|
||||
assert_eq!(swmr.data_end(0, 17_857), Ok(2048));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
Reference in New Issue
Block a user