quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

format: bound and batch every chunk fetch over Storage

Browse Source 
Only the full read split its chunk fetches into 64 MiB batches. The
selection path, the indexed read and the parallel_read decoders fetched
every chunk's stored bytes in one read_ranges call, each extent bounded only
by the file length, so a crafted chunk index pointing many chunks at one
large extent made File::open_storage hold chunks x extent bytes (3.3 GB from
a 16.8 MB file) before the first decode error.

- storage::for_each_extent_batch is now the one way raw-data reads fetch
  chunk bytes: batches of at most RAW_BATCH_BYTES (now pub), each decoded
  before the next is fetched. Used by the full, cached, indexed, selection
  and parallel_read paths; the sweep read uses read_extent per chunk.
- ExtentReq carries each chunk's claimed extent (bounds-checked as before,
  same errors) and the prefix actually fetched:
  filters::stored_chunk_limit — the chunk size if unfiltered, else each
  applied filter's worst-case growth (n + n/4 + 4096 per codec; unbounded
  only for an application-registered codec). The in-memory path cuts the
  slice it decodes the same way, so both paths still agree.
- tests/raw_fetch_bounds.rs: a crafted chunked_large.h5 (ten chunks all
  claiming 20 MiB at one padding blob) read through every path over a
  storage that records the largest single fetch; and 160 MiB of legitimate
  unfiltered chunks fetched batch by batch. Before: one 80 MiB fetch
  (selection) and one 160 MiB fetch; after: within the budget.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-26 18:30:30 -05:00
co-authored by Claude Opus 5.5
parent f191dc09d5
commit 7d629f49e3
10 changed files with 825 additions and 285 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/partial_read.rs
+45 -45
View File
@@ -26,7 +26,7 @@ use crate::error::FormatError;
use crate::filter_pipeline::FilterPipeline;
use crate::filters::{all_filters_skipped, decompress_chunk_exact_with};
use crate::selection::Selection;
use crate::storage::{ExtentBytes, Storage};
use crate::storage::{ExtentReq, Storage, for_each_extent_batch};
/// The smallest axis-aligned box containing every selected element, as
/// `(start, extent)` per dimension. `None` when there is nothing to gain or
@@ -373,50 +373,50 @@ pub fn read_selection_in<S: Storage + ?Sized>(
})
})
.collect();
// Their stored bytes, in one batch when the file is not in memory.
let extents: Vec<(u64, usize, bool)> = if file_data.as_contiguous().is_some() {
Vec::new()
} else {
wanted
.iter()
.map(|c| (c.address, c.chunk_size as usize, true))
.collect()
};
let raw_bytes = ExtentBytes::fetch(file_data, &extents)?;
// Chunks are decoded into this thread's reusable buffers.
crate::chunked_read::with_scratch(|scratch| -> Result<(), FormatError> {
for (i, chunk) in wanted.iter().enumerate() {
let origin = &chunk.offsets[..rank];
usize::try_from(chunk.address)
.map_err(|_| FormatError::Overflow("chunk address exceeds usize".into()))?;
let raw = raw_bytes.get(i, chunk.address, chunk.chunk_size as usize)?;
// Mirrors the full-read path: filter-mask bit i set means
// filter i was not applied to this chunk.
let data: &[u8] = match pipeline {
Some(pl) if !all_filters_skipped(pl, chunk.filter_mask) => {
decompress_chunk_exact_with(
raw,
pl,
chunk_bytes,
elem_size as u32,
chunk.filter_mask,
&chunk.offsets[..rank],
scratch,
)?
}
_ => raw,
};
copy_overlap(
data,
origin,
&chunk_shape,
&mut boxed,
&box_start,
&box_extent,
elem_size,
);
}
Ok(())
// Their stored bytes, batch by batch when the file is not in
// memory; each batch's chunks are decoded into this thread's
// reusable buffers before the next batch is fetched.
let reqs: Vec<ExtentReq> = wanted
.iter()
.map(|c| crate::chunked_read::chunk_req(c, pipeline, chunk_bytes, true))
.collect();
for_each_extent_batch(file_data, &reqs, |batch, raw_bytes| {
crate::chunked_read::with_scratch(|scratch| -> Result<(), FormatError> {
for i in batch {
let chunk = wanted[i];
let origin = &chunk.offsets[..rank];
usize::try_from(chunk.address).map_err(|_| {
FormatError::Overflow("chunk address exceeds usize".into())
})?;
let raw = raw_bytes.get(i, &reqs[i])?;
// Mirrors the full-read path: filter-mask bit i set
// means filter i was not applied to this chunk.
let data: &[u8] = match pipeline {
Some(pl) if !all_filters_skipped(pl, chunk.filter_mask) => {
decompress_chunk_exact_with(
raw,
pl,
chunk_bytes,
elem_size as u32,
chunk.filter_mask,
&chunk.offsets[..rank],
scratch,
)?
}
_ => raw,
};
copy_overlap(
data,
origin,
&chunk_shape,
&mut boxed,
&box_start,
&box_extent,
elem_size,
);
}
Ok(())
})
})?;
}
_ => return Ok(None),