wasm: sizes a server or a dataset names are errors, not aborts
A read longer than isize::MAX (2 GiB on wasm32) aborted the module in LazyStorage::assemble (capacity_overflow), taking every open file on the page with it, and a hostile server only had to claim a large length and serve a heap collection of 2 GiB + 4 KiB to get there (after fetching 2 GiB). Reading a large u8 dataset whole aborted the same way when its values were widened to 64 bits. - LazyConfig::max_fetch (openUrl option maxFetch, default 512 MiB, at most 1 GiB): a read longer than it fails at once, before anything is fetched, and an operation whose passes would fetch more than it fails before fetching (Operation::charge). assemble reserves fallibly. - Reader::read refuses a read that would use more than 1 GiB while decoding (core::MAX_READ_BYTES: stored bytes + 64-bit values + result) with an error naming readHyperslab, before reading. - openUrl refuses a file of 4 GiB or more at open on wasm32: the format code turns offsets into usize, so nothing past 4 GiB can be read there (shown by a new test: data at 3 GiB reads, a 4 GiB file is refused). maxDownload is bounded to 1 GiB. Tests: make_fixture.py writes limits.h5 (a sparse 2^28 + 1024 byte u8 dataset), hostile_vl.h5 (the reviewer's collection) and far.h5 (data at 3 GiB); test.mjs (wasm32) and tests/lazy.rs (native) check each is an error or reads, and that the module survives. Before: RuntimeError: unreachable in Node; the native test read the huge dataset and fetched 2 GiB. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
@@ -14,6 +14,15 @@ use clawhdf5_format::datatype::{Datatype, DatatypeByteOrder};
|
||||
use clawhdf5_format::storage::Storage;
|
||||
use clawhdf5_format::vl_data::{VlResolver, check_element_size};
|
||||
|
||||
/// The most memory one read may use while it decodes: the stored bytes,
|
||||
/// the values at 64 bits (integers are widened first) and the values
|
||||
/// returned. A larger read fails with an error naming `readHyperslab`,
|
||||
/// before anything is read: on wasm32 a buffer past 2 GiB cannot be
|
||||
/// allocated at all, and failing to allocate aborts the module (every open
|
||||
/// file on the page with it). 1 GiB leaves room in wasm32's 4 GiB for the
|
||||
/// file's cached blocks and the JavaScript copy of the result.
|
||||
pub const MAX_READ_BYTES: u64 = 1 << 30;
|
||||
|
||||
/// Errors are reported to JavaScript as messages.
|
||||
pub type Result<T> = std::result::Result<T, String>;
|
||||
|
||||
@@ -241,13 +250,27 @@ impl Reader {
|
||||
if let Datatype::VariableLength { size, .. } = array_base(&dt) {
|
||||
check_element_size(*size, self.file.superblock().offset_size).map_err(err)?;
|
||||
}
|
||||
let raw = ds.read_selection(&selection).map_err(err)?;
|
||||
let data = self.decode(&raw, &dt)?;
|
||||
out_shape.extend(element_shape(&dt));
|
||||
let expected = out_shape
|
||||
.iter()
|
||||
.try_fold(1u64, |acc, &d| acc.checked_mul(d))
|
||||
.ok_or("selection size overflows")?;
|
||||
let cost = expected.saturating_mul(bytes_per_value(&dt));
|
||||
if cost > MAX_READ_BYTES {
|
||||
return Err(format!(
|
||||
"reading {path}{} would take about {} MiB of memory, more than the {} MiB \
|
||||
one read may use; read it in parts (readHyperslab)",
|
||||
if slab.is_some() {
|
||||
" (this selection)"
|
||||
} else {
|
||||
" whole"
|
||||
},
|
||||
cost >> 20,
|
||||
MAX_READ_BYTES >> 20
|
||||
));
|
||||
}
|
||||
let raw = ds.read_selection(&selection).map_err(err)?;
|
||||
let data = self.decode(&raw, &dt)?;
|
||||
if data.len() as u64 != expected {
|
||||
return Err(format!(
|
||||
"read {} values for shape {out_shape:?} ({expected} expected)",
|
||||
@@ -316,6 +339,27 @@ impl Reader {
|
||||
}
|
||||
}
|
||||
|
||||
/// Memory one value of type `dt` takes while [`Reader::read`] decodes it
|
||||
/// (an array type's elements count as values): its stored bytes, plus what
|
||||
/// [`Reader::decode`] builds from them. A string counts its `String` (24
|
||||
/// bytes on 64-bit targets, less on wasm32) and, for a fixed-length one,
|
||||
/// its text; a variable-length string's text lives in the heap and is
|
||||
/// bounded by the storage's own read limit.
|
||||
fn bytes_per_value(dt: &Datatype) -> u64 {
|
||||
let base = array_base(dt);
|
||||
let stored = u64::from(base.type_size());
|
||||
stored
|
||||
+ match base {
|
||||
Datatype::FloatingPoint { size, .. } if *size <= 4 => 4,
|
||||
Datatype::FloatingPoint { .. } => 8,
|
||||
// Widened to 64 bits, then narrowed to a new vector.
|
||||
Datatype::FixedPoint { .. } => 8 + stored,
|
||||
Datatype::String { .. } => 24 + stored,
|
||||
Datatype::VariableLength { .. } | Datatype::Enumeration { .. } => 24,
|
||||
_ => 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// Narrow integers read at 64 bits to the dataset's own width. The source is
|
||||
/// that width, so this cannot fail on correct input; it is checked anyway.
|
||||
fn narrow<S: Copy + std::fmt::Display, T: TryFrom<S>>(v: Vec<S>) -> Result<Vec<T>> {
|
||||
|
||||
@@ -42,6 +42,10 @@ use clawhdf5_format::storage::Storage;
|
||||
/// `docs/design/range-reads.md` §2 measured).
|
||||
pub const DEFAULT_BLOCK_SIZE: u64 = 1 << 20;
|
||||
|
||||
/// Default of [`LazyConfig::max_fetch`]: 512 MiB, the same as `openUrl`'s
|
||||
/// `maxDownload` for a server without range support.
|
||||
pub const DEFAULT_MAX_FETCH: u64 = 512 << 20;
|
||||
|
||||
/// The message of the error a read that misses returns. It never reaches
|
||||
/// the caller of [`LazyStorage::attempt`]: a pass that missed is re-run.
|
||||
pub const NEED_BYTES: &str = "bytes not fetched yet (restartable read)";
|
||||
@@ -58,6 +62,14 @@ pub struct LazyConfig {
|
||||
/// Largest single range asked for, in bytes (whole blocks, at least
|
||||
/// one); longer runs are split so they can be fetched in parallel.
|
||||
pub max_request: u64,
|
||||
/// Most bytes one operation may fetch (at least one block), and so the
|
||||
/// longest single read: a read longer than this fails at once, before
|
||||
/// anything is fetched, and so does an operation whose passes would
|
||||
/// fetch more. The file's length comes from the server, so without
|
||||
/// this a hostile file (a heap "collection" claiming 2 GiB) makes the
|
||||
/// reader fetch and hold whatever it names; on wasm32 a buffer past
|
||||
/// 2 GiB cannot even be allocated.
|
||||
pub max_fetch: u64,
|
||||
}
|
||||
|
||||
impl Default for LazyConfig {
|
||||
@@ -66,6 +78,7 @@ impl Default for LazyConfig {
|
||||
block_size: DEFAULT_BLOCK_SIZE,
|
||||
capacity: 64 << 20,
|
||||
max_request: 8 << 20,
|
||||
max_fetch: DEFAULT_MAX_FETCH,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -137,6 +150,28 @@ fn lock(m: &Mutex<State>) -> MutexGuard<'_, State> {
|
||||
/// [`LazyStorage::operation`].
|
||||
pub struct Operation<'a> {
|
||||
storage: &'a LazyStorage,
|
||||
/// Bytes fetched for this operation so far.
|
||||
fetched: std::cell::Cell<u64>,
|
||||
}
|
||||
|
||||
impl Operation<'_> {
|
||||
/// Count `ranges` against the operation's budget
|
||||
/// ([`LazyConfig::max_fetch`]) before they are fetched: an error, and
|
||||
/// nothing counted, if they would take it past the budget.
|
||||
pub fn charge(&self, ranges: &[Range<u64>]) -> Result<(), String> {
|
||||
let max = self.storage.config.max_fetch;
|
||||
let total = ranges.iter().fold(self.fetched.get(), |n, r| {
|
||||
n.saturating_add(r.end.saturating_sub(r.start))
|
||||
});
|
||||
if total > max {
|
||||
return Err(format!(
|
||||
"this call would fetch more than {max} bytes of the file (the maxFetch limit); \
|
||||
read less at a time (readHyperslab) or raise maxFetch"
|
||||
));
|
||||
}
|
||||
self.fetched.set(total);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Operation<'_> {
|
||||
@@ -154,6 +189,7 @@ impl LazyStorage {
|
||||
pub fn new(len: u64, mut config: LazyConfig) -> Self {
|
||||
config.block_size = config.block_size.max(512);
|
||||
config.max_request = (config.max_request / config.block_size).max(1) * config.block_size;
|
||||
config.max_fetch = config.max_fetch.max(config.block_size);
|
||||
LazyStorage {
|
||||
len,
|
||||
config,
|
||||
@@ -180,7 +216,10 @@ impl LazyStorage {
|
||||
/// Hold it across every pass of one operation.
|
||||
pub fn operation(&self) -> Operation<'_> {
|
||||
lock(&self.state).active += 1;
|
||||
Operation { storage: self }
|
||||
Operation {
|
||||
storage: self,
|
||||
fetched: std::cell::Cell::new(0),
|
||||
}
|
||||
}
|
||||
|
||||
/// Run one pass of `f` over this storage. `Done` when `f` read nothing
|
||||
@@ -268,11 +307,12 @@ impl LazyStorage {
|
||||
mut f: impl FnMut() -> T,
|
||||
mut fetch: impl FnMut(Range<u64>) -> Result<Vec<u8>, String>,
|
||||
) -> Result<T, String> {
|
||||
let _op = self.operation();
|
||||
let op = self.operation();
|
||||
loop {
|
||||
match self.attempt(&mut f) {
|
||||
Step::Done(v) => return Ok(v),
|
||||
Step::Need(ranges) => {
|
||||
op.charge(&ranges)?;
|
||||
for r in ranges {
|
||||
let bytes = fetch(r.clone())?;
|
||||
self.supply_range(&r, &bytes)?;
|
||||
@@ -395,9 +435,35 @@ impl LazyStorage {
|
||||
Ok(have)
|
||||
}
|
||||
|
||||
fn assemble(&self, offset: u64, end: u64, blocks: &HashMap<u64, Arc<[u8]>>) -> Vec<u8> {
|
||||
/// Refuse a read of `n` bytes longer than an operation may fetch
|
||||
/// ([`LazyConfig::max_fetch`]), before its blocks are asked for.
|
||||
fn check_len(&self, n: u64) -> Result<(), FormatError> {
|
||||
let max = self.config.max_fetch;
|
||||
if n > max {
|
||||
return Err(FormatError::Storage(format!(
|
||||
"a read of {n} bytes is more than one call may fetch ({max} bytes, the maxFetch limit)"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The bytes `offset..end` from `blocks`, which hold every block of
|
||||
/// that span. The buffer is reserved fallibly: a length the address
|
||||
/// space cannot hold (past `isize::MAX` on wasm32) is an error, never
|
||||
/// an abort.
|
||||
fn assemble(
|
||||
&self,
|
||||
offset: u64,
|
||||
end: u64,
|
||||
blocks: &HashMap<u64, Arc<[u8]>>,
|
||||
) -> Result<Vec<u8>, FormatError> {
|
||||
let bs = self.config.block_size;
|
||||
let mut out = Vec::with_capacity(usize::try_from(end - offset).unwrap_or(0));
|
||||
let n = end - offset;
|
||||
let too_long =
|
||||
|| FormatError::Storage(format!("cannot hold a read of {n} bytes in memory"));
|
||||
let mut out = Vec::new();
|
||||
out.try_reserve_exact(usize::try_from(n).map_err(|_| too_long())?)
|
||||
.map_err(|_| too_long())?;
|
||||
let mut pos = offset;
|
||||
while pos < end {
|
||||
let i = pos / bs;
|
||||
@@ -407,7 +473,7 @@ impl LazyStorage {
|
||||
out.extend_from_slice(&block[from..to]);
|
||||
pos = i * bs + to as u64;
|
||||
}
|
||||
out
|
||||
Ok(out)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -416,10 +482,11 @@ impl Storage for LazyStorage {
|
||||
let Some(span) = self.span(offset, len as u64) else {
|
||||
return Ok(Cow::Owned(Vec::new()));
|
||||
};
|
||||
let end = offset.saturating_add(len as u64).min(self.len);
|
||||
self.check_len(end - offset)?;
|
||||
let metadata = len as u64 <= self.config.block_size;
|
||||
let blocks = self.blocks(std::slice::from_ref(&span), metadata)?;
|
||||
let end = offset.saturating_add(len as u64).min(self.len);
|
||||
Ok(Cow::Owned(self.assemble(offset, end, &blocks)))
|
||||
Ok(Cow::Owned(self.assemble(offset, end, &blocks)?))
|
||||
}
|
||||
|
||||
fn len(&self) -> u64 {
|
||||
@@ -428,26 +495,29 @@ impl Storage for LazyStorage {
|
||||
|
||||
fn read_ranges(&self, ranges: &[Range<u64>]) -> Result<Vec<Cow<'_, [u8]>>, FormatError> {
|
||||
let mut spans = Vec::with_capacity(ranges.len());
|
||||
let mut total = 0u64;
|
||||
for r in ranges {
|
||||
if r.end < r.start {
|
||||
return Err(FormatError::Storage(
|
||||
"read range ends before it starts".into(),
|
||||
));
|
||||
}
|
||||
total = total.saturating_add(r.end.min(self.len).saturating_sub(r.start));
|
||||
spans.extend(self.span(r.start, r.end - r.start));
|
||||
}
|
||||
self.check_len(total)?;
|
||||
let blocks = self.blocks(&spans, false)?;
|
||||
Ok(ranges
|
||||
ranges
|
||||
.iter()
|
||||
.map(|r| {
|
||||
let end = r.end.min(self.len);
|
||||
if r.start >= end {
|
||||
Cow::Owned(Vec::new())
|
||||
Ok(Cow::Owned(Vec::new()))
|
||||
} else {
|
||||
Cow::Owned(self.assemble(r.start, end, &blocks))
|
||||
self.assemble(r.start, end, &blocks).map(Cow::Owned)
|
||||
}
|
||||
})
|
||||
.collect())
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -465,6 +535,7 @@ mod tests {
|
||||
block_size: block,
|
||||
capacity,
|
||||
max_request: 4 * block,
|
||||
max_fetch: DEFAULT_MAX_FETCH,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -654,6 +725,57 @@ mod tests {
|
||||
assert_eq!(s.stats().requests, before.requests);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_read_longer_than_max_fetch_fails_without_fetching() {
|
||||
// A hostile file names a 2 GiB heap collection in a "file" the
|
||||
// server claims is 1 TiB: the read is refused before any block is
|
||||
// asked for (on wasm32 its buffer could not even be allocated).
|
||||
let s = LazyStorage::new(1 << 40, LazyConfig::default());
|
||||
let step = s.attempt(|| s.read_at(4096, (1usize << 31) + 4096).map(|b| b.len()));
|
||||
match step {
|
||||
Step::Done(Err(e)) => assert!(e.to_string().contains("maxFetch"), "{e}"),
|
||||
other => panic!("expected a refusal, got {other:?}"),
|
||||
}
|
||||
let step = s.attempt(|| s.read_ranges(&[0..(600 << 20)]).map(|v| v.len()));
|
||||
assert!(matches!(step, Step::Done(Err(_))), "{step:?}");
|
||||
assert_eq!(s.stats().requests, 0);
|
||||
// At the limit it is an ordinary miss.
|
||||
let s = LazyStorage::new(1 << 40, config(1024, 1 << 20));
|
||||
let step = s.attempt(|| s.read_at(0, DEFAULT_MAX_FETCH as usize).map(|b| b.len()));
|
||||
assert!(matches!(step, Step::Need(_)), "{step:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_operation_stops_at_its_fetch_budget() {
|
||||
// Many small reads, none over the limit, that together would fetch
|
||||
// more than the budget: the operation fails before fetching past it.
|
||||
let data = file(64 * 1024);
|
||||
let mut c = config(1024, 1 << 20);
|
||||
c.max_fetch = 8 * 1024;
|
||||
let s = LazyStorage::new(data.len() as u64, c);
|
||||
let e = s
|
||||
.run_blocking(
|
||||
|| {
|
||||
(0..64)
|
||||
.map(|i| owned(s.read_at(i * 1024, 8)))
|
||||
.collect::<Result<Vec<_>, _>>()
|
||||
},
|
||||
|r| Ok(data[r.start as usize..r.end as usize].to_vec()),
|
||||
)
|
||||
.unwrap_err();
|
||||
assert!(e.contains("maxFetch"), "{e}");
|
||||
assert!(s.stats().bytes_fetched <= 8 * 1024, "{:?}", s.stats());
|
||||
// Within the budget it completes, and the budget is per operation.
|
||||
for _ in 0..3 {
|
||||
s.run_blocking(
|
||||
|| owned(s.read_at(10 * 1024, 3000)),
|
||||
|r| Ok(data[r.start as usize..r.end as usize].to_vec()),
|
||||
)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_failed_fetch_is_an_error_not_data() {
|
||||
let data = file(4096);
|
||||
|
||||
@@ -389,11 +389,14 @@ impl Http {
|
||||
storage: &LazyStorage,
|
||||
mut f: impl FnMut() -> T,
|
||||
) -> Result<T, JsError> {
|
||||
let _op = storage.operation();
|
||||
let op = storage.operation();
|
||||
loop {
|
||||
match storage.attempt(&mut f) {
|
||||
Step::Done(v) => return Ok(v),
|
||||
Step::Need(ranges) => self.fetch(storage, &ranges).await?,
|
||||
Step::Need(ranges) => {
|
||||
op.charge(&ranges).map_err(js_err)?;
|
||||
self.fetch(storage, &ranges).await?
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -426,6 +429,24 @@ fn int_opt(opts: &JsValue, key: &str, min: f64, max: f64) -> Result<Option<u64>,
|
||||
}
|
||||
}
|
||||
|
||||
/// Most bytes `maxFetch` and `maxDownload` may allow: 1 GiB. wasm32 has
|
||||
/// 4 GiB of memory and no buffer past 2 GiB, and what is fetched is held
|
||||
/// while it is decoded.
|
||||
const MAX_FETCH_LIMIT: u64 = 1 << 30;
|
||||
|
||||
/// The largest file `openUrl` reads by ranges: on wasm32, 4 GiB - 1 bytes.
|
||||
/// The format code turns file offsets into `usize` to use them (with a
|
||||
/// clean error past it, see scripts/check-32bit-casts.sh), so on a 32-bit
|
||||
/// target nothing at 4 GiB or beyond can be read; a larger file is refused
|
||||
/// at open rather than failing on whichever read reaches past 4 GiB. On
|
||||
/// 64-bit targets it is 2^53 - 1, the largest offset a JavaScript number
|
||||
/// holds exactly.
|
||||
const MAX_REMOTE_LENGTH: u64 = if (usize::MAX as u64) < MAX_SAFE_INTEGER as u64 {
|
||||
usize::MAX as u64
|
||||
} else {
|
||||
MAX_SAFE_INTEGER as u64
|
||||
};
|
||||
|
||||
fn config_from(opts: &JsValue) -> Result<LazyConfig, JsError> {
|
||||
let mut c = LazyConfig::default();
|
||||
if let Some(b) = int_opt(opts, "blockSize", 512.0, (64u64 << 20) as f64)? {
|
||||
@@ -434,6 +455,12 @@ fn config_from(opts: &JsValue) -> Result<LazyConfig, JsError> {
|
||||
if let Some(n) = int_opt(opts, "cacheSize", 0.0, MAX_SAFE_INTEGER)? {
|
||||
c.capacity = n;
|
||||
}
|
||||
if let Some(n) = int_opt(opts, "maxFetch", 512.0, MAX_FETCH_LIMIT as f64)? {
|
||||
c.max_fetch = n;
|
||||
}
|
||||
// Read by remote.js; checked here so a value wasm32 cannot hold is an
|
||||
// option error rather than a download that cannot be kept.
|
||||
int_opt(opts, "maxDownload", 0.0, MAX_FETCH_LIMIT as f64)?;
|
||||
Ok(c)
|
||||
}
|
||||
|
||||
@@ -444,15 +471,21 @@ fn config_from(opts: &JsValue) -> Result<LazyConfig, JsError> {
|
||||
/// `opts` (all optional):
|
||||
/// - `blockSize` — bytes per request block, 512 to 64 MiB (default 1 MiB);
|
||||
/// - `cacheSize` — bytes of blocks kept between calls (default 64 MiB);
|
||||
/// - `maxFetch` — most bytes one call may fetch, and so the longest single
|
||||
/// read, up to 1 GiB (default 512 MiB): a call that would fetch more
|
||||
/// fails before fetching it;
|
||||
/// - `fallback` — `"download"` (default) reads the whole file when the
|
||||
/// server ignores `Range` (answers 200), up to `maxDownload` bytes
|
||||
/// (default 512 MiB); `"error"` refuses such a server;
|
||||
/// (default 512 MiB, at most 1 GiB); `"error"` refuses such a server;
|
||||
/// - `headers`, `credentials` — passed to every `fetch`;
|
||||
/// - `parallel` — range requests in flight at once (default 6);
|
||||
/// - `fetch` — a `fetch`-compatible function to use instead of the global.
|
||||
///
|
||||
/// Cross-origin servers must allow CORS and expose `Content-Range` (or
|
||||
/// answer `HEAD` with `Content-Length`).
|
||||
/// answer `HEAD` with `Content-Length`). A file may be up to 4 GiB - 1
|
||||
/// bytes long (wasm32 offsets); a longer one is refused at open. A whole-dataset `read` that would use more than 1 GiB of
|
||||
/// memory ([`core::MAX_READ_BYTES`]) is refused: read it in parts with
|
||||
/// `readHyperslab`.
|
||||
#[wasm_bindgen(js_name = openUrl)]
|
||||
pub async fn open_url(url: String, opts: JsValue) -> Result<RemoteFile, JsError> {
|
||||
let config = config_from(&opts)?;
|
||||
@@ -477,6 +510,12 @@ pub async fn open_url(url: String, opts: JsValue) -> Result<RemoteFile, JsError>
|
||||
.as_f64()
|
||||
.filter(|x| x.fract() == 0.0 && (0.0..=MAX_SAFE_INTEGER).contains(x))
|
||||
.ok_or_else(|| js_err(format!("{url}: the server gave no usable file size")))?;
|
||||
if length as u64 > MAX_REMOTE_LENGTH {
|
||||
return Err(js_err(format!(
|
||||
"{url} is {length} bytes; openUrl reads files of up to {MAX_REMOTE_LENGTH} bytes \
|
||||
(4 GiB - 1: the WebAssembly reader addresses a file with 32-bit offsets)"
|
||||
)));
|
||||
}
|
||||
let http = Http {
|
||||
url,
|
||||
opts,
|
||||
|
||||
Reference in New Issue
Block a user