quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases 1 Wiki Activity

clawhdf5-remote tests: open + list and a dataset read counted apart

Browse Source 
The per-file report now separates a tree view (open, every group's
entries, every dataset's shape and type) from reading the largest
dataset under 64 MiB, and checks the budget the design's testing section
asks for: listing the IMERG file (file A of docs/design/range-reads.md
section 2) takes at most 3 requests when CLAWHDF5_REMOTE_CORPUS includes
it. The test server now counts a response's bytes before sending it: a
client could read a body and reset the counters before the server thread
had added it, so the counts of the next file were occasionally too high.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-26 17:26:57 -05:00
co-authored by Claude Opus 5.5
parent c513f7e6d7
commit ebe51f8e97
3 changed files with 69 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-remote/tests/http.rs
+36 -22
View File
@@ -53,7 +53,7 @@ fn compare(files: &[std::path::PathBuf], report: bool) -> (usize, usize) {
.collect();
let server = Server::start(served.clone());
let (mut same, mut refused) = (0, 0);
let mut totals = [0u64; 5];
let mut totals = [0u64; 7];
for (i, p) in files.iter().enumerate() {
let Some((url_path, bytes)) = served
.iter()
@@ -104,45 +104,59 @@ fn compare(files: &[std::path::PathBuf], report: bool) -> (usize, usize) {
assert!(storage.stats().reads > 0, "read through the cache");
same += 1;
// Cost of "list + read one dataset": with the block cache, and
// with none (every read a request).
let with = storage_for_url(&url, &quick()).unwrap();
// Cost of "open + list" (a tree view) and of then reading the
// largest dataset (a plot): with the block cache, and with none
// (every read a request). Requests and bytes as the server saw
// them, the open included.
server.reset();
if let Ok(f) = File::open_storage(with.clone()) {
list_and_read_one(&f);
let with = storage_for_url(&url, &quick()).unwrap();
let f = File::open_storage(with.clone()).unwrap();
let pick = common::list(&f);
let (list_requests, list_bytes) = (server.requests(), server.bytes());
if let Some((addr, _)) = pick {
common::read_one(&f, addr);
}
let (cached_requests, cached_bytes) = (server.requests(), server.bytes());
let (bare, _) = HttpStorage::open(&url, quick().http).unwrap();
let bare = Arc::new(bare);
server.reset();
if let Ok(f) = File::open_storage(bare.clone()) {
let (bare, _) = HttpStorage::open(&url, quick().http).unwrap();
if let Ok(f) = File::open_storage(Arc::new(bare)) {
list_and_read_one(&f);
}
let uncached_requests = server.requests();
totals[0] += 1 + cached_requests; // + the open probe
totals[1] += cached_bytes + with.config().block_size.min(bytes.len() as u64);
totals[2] += 1 + uncached_requests;
totals[3] += bytes.len() as u64;
totals[4] += 1;
for (t, v) in totals.iter_mut().zip([
list_requests,
list_bytes,
cached_requests,
cached_bytes,
uncached_requests,
bytes.len() as u64,
1,
]) {
*t += v;
}
if report {
eprintln!(
"requests cached {:>6} uncached {:>8} bytes {:>12} of {:>12} {}",
1 + cached_requests,
1 + uncached_requests,
cached_bytes + with.config().block_size.min(bytes.len() as u64),
"open+list {list_requests:>4} req {list_bytes:>10} B | +read {:>5} req \
{cached_bytes:>10} B | uncached {uncached_requests:>7} req | file {:>10} B {}",
cached_requests,
bytes.len(),
p.display()
);
}
// Files within one block: opening fetched everything.
if bytes.len() as u64 <= with.config().block_size {
assert_eq!(cached_requests, 0, "{}", p.display());
assert_eq!(cached_requests, 1, "{}", p.display());
}
// The budget of docs/design/range-reads.md (Testing): listing the
// IMERG file (file A of section 2) takes at most 3 requests.
if p.ends_with("xarray-data/imerghh_730.hdf5") {
assert!(list_requests <= 3, "{}: {list_requests}", p.display());
}
}
eprintln!(
"list + read one dataset over {} files: {} requests with the 1 MiB block cache \
({} bytes transferred, files {} bytes), {} requests without a cache",
totals[4], totals[0], totals[1], totals[3], totals[2]
"{} files ({} bytes): open + list {} requests, {} bytes; + read the largest \
dataset {} requests, {} bytes (1 MiB block cache); without a cache {} requests",
totals[6], totals[5], totals[0], totals[1], totals[2], totals[3], totals[4]
);
(same, refused)
}