quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases 1 Wiki Activity

clawhdf5-remote tests: open + list and a dataset read counted apart

Browse Source 
The per-file report now separates a tree view (open, every group's
entries, every dataset's shape and type) from reading the largest
dataset under 64 MiB, and checks the budget the design's testing section
asks for: listing the IMERG file (file A of docs/design/range-reads.md
section 2) takes at most 3 requests when CLAWHDF5_REMOTE_CORPUS includes
it. The test server now counts a response's bytes before sending it: a
client could read a body and reset the counters before the server thread
had added it, so the counts of the next file were occasionally too high.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-26 17:26:57 -05:00
co-authored by Claude Opus 5.5
parent c513f7e6d7
commit ebe51f8e97
3 changed files with 69 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-remote/tests/common/mod.rs
+30 -14
View File
@@ -147,11 +147,13 @@ fn dataset(out: &mut String, path: &str, ds: &clawhdf5::Dataset<'_>) {
}
}
/// Open, list every group, and read the first dataset found whose data is
/// at most `MAX_DATA_BYTES` (a tree view plus one plot).
pub fn list_and_read_one(file: &File) {
/// List the file as a tree view does — every group's entries, every
/// dataset's shape and type — and return the largest dataset whose data
/// is at most `MAX_DATA_BYTES` (address, bytes), the one a viewer would
/// plot.
pub fn list(file: &File) -> Option<(u64, u64)> {
let mut seen = HashSet::new();
let mut read_one = false;
let mut largest: Option<(u64, u64)> = None;
let mut queue = VecDeque::from([file.superblock().root_group_address]);
while let Some(addr) = queue.pop_front() {
if seen.len() >= MAX_OBJECTS || !seen.insert(addr) {
@@ -160,22 +162,36 @@ pub fn list_and_read_one(file: &File) {
let group = file.group_at(addr);
if let Ok(ds) = file.dataset_at(addr) {
let _ = (ds.shape(), ds.dtype());
if !read_one {
let small = ds.shape().ok().and_then(|s| {
let n = s.iter().try_fold(1u64, |a, &d| a.checked_mul(d))?;
let size = u64::from(ds.raw_datatype().ok()?.type_size());
n.checked_mul(size).filter(|&b| b <= MAX_DATA_BYTES)
});
if small.is_some() {
let _ = ds.read_selection(&Selection::All);
read_one = true;
}
let bytes = ds.shape().ok().and_then(|s| {
let n = s.iter().try_fold(1u64, |a, &d| a.checked_mul(d))?;
let size = u64::from(ds.raw_datatype().ok()?.type_size());
n.checked_mul(size).filter(|&b| b <= MAX_DATA_BYTES)
});
if let Some(b) = bytes
&& largest.is_none_or(|(_, l)| b > l)
{
largest = Some((addr, b));
}
}
if let Ok(entries) = group.entries() {
queue.extend(entries.into_iter().map(|(_, a)| a));
}
}
largest
}
/// Read the dataset at `addr` whole.
pub fn read_one(file: &File, addr: u64) {
if let Ok(ds) = file.dataset_at(addr) {
let _ = ds.read_selection(&Selection::All);
}
}
/// [`list`], then [`read_one`] of the dataset it picks.
pub fn list_and_read_one(file: &File) {
if let Some((addr, _)) = list(file) {
read_one(file, addr);
}
}
/// External virtual-dataset sources read from `dir`, as `File::open` finds