wasm: sizes a server or a dataset names are errors, not aborts
A read longer than isize::MAX (2 GiB on wasm32) aborted the module in LazyStorage::assemble (capacity_overflow), taking every open file on the page with it, and a hostile server only had to claim a large length and serve a heap collection of 2 GiB + 4 KiB to get there (after fetching 2 GiB). Reading a large u8 dataset whole aborted the same way when its values were widened to 64 bits. - LazyConfig::max_fetch (openUrl option maxFetch, default 512 MiB, at most 1 GiB): a read longer than it fails at once, before anything is fetched, and an operation whose passes would fetch more than it fails before fetching (Operation::charge). assemble reserves fallibly. - Reader::read refuses a read that would use more than 1 GiB while decoding (core::MAX_READ_BYTES: stored bytes + 64-bit values + result) with an error naming readHyperslab, before reading. - openUrl refuses a file of 4 GiB or more at open on wasm32: the format code turns offsets into usize, so nothing past 4 GiB can be read there (shown by a new test: data at 3 GiB reads, a 4 GiB file is refused). maxDownload is bounded to 1 GiB. Tests: make_fixture.py writes limits.h5 (a sparse 2^28 + 1024 byte u8 dataset), hostile_vl.h5 (the reviewer's collection) and far.h5 (data at 3 GiB); test.mjs (wasm32) and tests/lazy.rs (native) check each is an error or reads, and that the module survives. Before: RuntimeError: unreachable in Node; the native test read the huge dataset and fetched 2 GiB. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
@@ -193,6 +193,7 @@ fn config(block: u64) -> LazyConfig {
|
||||
// A small budget, so eviction between operations is exercised.
|
||||
capacity: 16 * block,
|
||||
max_request: 8 * block,
|
||||
..LazyConfig::default()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -310,6 +311,95 @@ fn h5py_and_netcdf4_files_read_the_same_lazily() {
|
||||
eprintln!("skipping: {} lacks h5py/netCDF4/numpy", python());
|
||||
return;
|
||||
}
|
||||
let dir = fixture_dir();
|
||||
for name in ["fixture.h5", "fixture.nc"] {
|
||||
let data = std::fs::read(dir.path().join(name)).unwrap();
|
||||
for block in [512, 64 * 1024] {
|
||||
let (_, _, lines) = check_equal(name, &data, block);
|
||||
assert!(lines.iter().filter(|l| l.contains(" read: Ok")).count() >= 2);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The bytes of `data` at `r`, zero past its end: a server that claims
|
||||
/// the file is longer than it is.
|
||||
fn fetch_padded(data: &[u8], r: Range<u64>) -> Result<Vec<u8>, String> {
|
||||
let mut out = vec![0u8; (r.end - r.start) as usize];
|
||||
let len = data.len() as u64;
|
||||
if r.start < len {
|
||||
let end = r.end.min(len);
|
||||
out[..(end - r.start) as usize].copy_from_slice(&data[r.start as usize..end as usize]);
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Sizes a hostile server or a large dataset can name are errors, never
|
||||
/// allocations that abort the wasm module: make_fixture.py's limits.h5 and
|
||||
/// hostile_vl.h5 (see write_limits there).
|
||||
#[test]
|
||||
fn size_limits_are_errors_not_aborts() {
|
||||
if !python_available() {
|
||||
assert!(
|
||||
!std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1"),
|
||||
"CLAWHDF5_REQUIRE_INTEROP=1 but {} lacks h5py/netCDF4/numpy",
|
||||
python()
|
||||
);
|
||||
eprintln!("skipping: {} lacks h5py/netCDF4/numpy", python());
|
||||
return;
|
||||
}
|
||||
let dir = fixture_dir();
|
||||
|
||||
// Read whole, /huge_u8 would widen 2^28 values to 64 bits (2 GiB): an
|
||||
// error naming readHyperslab, before its chunks are read. A window of
|
||||
// it reads.
|
||||
let data = std::fs::read(dir.path().join("limits.h5")).unwrap();
|
||||
let n = (1u64 << 28) + 1024;
|
||||
let window = Hyperslab {
|
||||
start: vec![n - 4],
|
||||
count: vec![4],
|
||||
stride: None,
|
||||
block: None,
|
||||
};
|
||||
let local = Reader::open(data.clone()).unwrap();
|
||||
let lazy = Lazy::open(data, LazyConfig::default()).unwrap();
|
||||
let before = lazy.storage.stats().requests;
|
||||
for e in [
|
||||
local.read("/huge_u8", None).unwrap_err(),
|
||||
lazy.call(|r| r.read("/huge_u8", None)).unwrap_err(),
|
||||
] {
|
||||
assert!(e.contains("readHyperslab"), "{e}");
|
||||
}
|
||||
assert_eq!(lazy.storage.stats().requests, before, "nothing fetched");
|
||||
for part in [
|
||||
local.read("/huge_u8", Some(&window)).unwrap(),
|
||||
lazy.call(|r| r.read("/huge_u8", Some(&window))).unwrap(),
|
||||
] {
|
||||
assert_eq!(format!("{:?}", part.data), "U8([0, 0, 0, 7])");
|
||||
}
|
||||
|
||||
// A server that claims 3 GiB and a heap collection of 2 GiB + 4 KiB:
|
||||
// reading the strings fails at once, fetching a few blocks.
|
||||
let data = std::fs::read(dir.path().join("hostile_vl.h5")).unwrap();
|
||||
let storage = Arc::new(LazyStorage::new(3 << 30, LazyConfig::default()));
|
||||
let s = storage.clone();
|
||||
let reader = storage
|
||||
.run_blocking(
|
||||
|| Reader::open_storage(s.clone()),
|
||||
|r| fetch_padded(&data, r),
|
||||
)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let e = storage
|
||||
.run_blocking(|| reader.read("/a", None), |r| fetch_padded(&data, r))
|
||||
.unwrap()
|
||||
.unwrap_err();
|
||||
assert!(e.contains("maxFetch"), "{e}");
|
||||
let st = storage.stats();
|
||||
assert!(st.requests <= 4 && st.bytes_fetched <= 4 << 20, "{st:?}");
|
||||
}
|
||||
|
||||
/// make_fixture.py's files, written to a temporary directory.
|
||||
fn fixture_dir() -> tempfile::TempDir {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let generator = Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("../../examples/wasm-viewer/test/make_fixture.py");
|
||||
@@ -323,13 +413,7 @@ fn h5py_and_netcdf4_files_read_the_same_lazily() {
|
||||
"{}",
|
||||
String::from_utf8_lossy(&out.stderr)
|
||||
);
|
||||
for name in ["fixture.h5", "fixture.nc"] {
|
||||
let data = std::fs::read(dir.path().join(name)).unwrap();
|
||||
for block in [512, 64 * 1024] {
|
||||
let (_, _, lines) = check_equal(name, &data, block);
|
||||
assert!(lines.iter().filter(|l| l.contains(" read: Ok")).count() >= 2);
|
||||
}
|
||||
}
|
||||
dir
|
||||
}
|
||||
|
||||
fn hdf5_files(dir: &Path, out: &mut Vec<PathBuf>) {
|
||||
|
||||
Reference in New Issue
Block a user