Remote reads (range-read M2/M3: open_storage, HTTP/S3) and near-complete in-place editing #18

Merged
osobh merged 52 commits from feat/p3-remote-editor into main 2026-09-27 11:15:23 +00:00
2 changed files with 41 additions and 0 deletions
Showing only changes of commit c54c64cc9b - Show all commits
+11
View File
@@ -523,6 +523,17 @@ impl File {
self.data.contiguous()
}
/// The bytes [`as_bytes`](Self::as_bytes) returns, as a [`Storage`],
/// for every backend: from the superblock on, bounded by the recorded
/// end of file, with a metadata cache image laid over them. Code that
/// parses the file itself with the `clawhdf5_format` `*_in` functions
/// reads through this, so it works on remote files too; for a file in
/// memory its [`Storage::as_contiguous`] is [`as_bytes`](Self::as_bytes)
/// (and every read a slice of it).
pub fn storage(&self) -> &(dyn Storage + Send + Sync) {
&self.data
}
/// The error of a metadata cache image libhdf5 cannot load, when the
/// file has one. Such a file opens, as in libhdf5, and every object
/// lookup fails with this error; code that parses [`Self::as_bytes`]
@@ -432,3 +432,33 @@ fn storage_backed_files_keep_their_zero_copy_views_only_in_memory() {
"as_bytes over a range storage must not answer"
);
}
/// `File::storage` is the view `as_bytes` gives, for every backend: the
/// user block skipped, bounded by the end of file, a cache image laid over.
#[test]
fn file_storage_is_the_as_bytes_view_for_every_backend() {
let root = Path::new(env!("CARGO_MANIFEST_DIR"));
let mut files = Vec::new();
hdf5_files(&root.join("tests/fixtures"), &mut files);
hdf5_files(&root.join("../clawhdf5-format/tests/fixtures"), &mut files);
let mut compared = 0;
for p in files {
let Ok(local) = File::open(&p) else { continue };
let bytes = std::fs::read(&p).unwrap();
let remote = File::open_storage(Arc::new(CountingStorage::new(bytes))).unwrap();
let want = local.as_bytes();
let view = local.storage();
assert_eq!(view.as_contiguous(), Some(want), "{}", p.display());
let got = remote.storage();
assert!(got.as_contiguous().is_none());
assert_eq!(got.len(), want.len() as u64, "{}", p.display());
assert_eq!(
&*got.read_at(0, want.len()).unwrap(),
want,
"{}",
p.display()
);
compared += 1;
}
assert!(compared >= 40, "{compared}");
}