docs: known-issues — cite PRs #27, #28, #29

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>

.gitea/workflows/ci.yml

@@ -40,7 +40,7 @@ jobs:
           python3 -m venv /opt/interop
           # maturin + pytest: ci-test.sh builds the Python package
           # (crates/clawhdf5-py) and runs its tests against h5py.
-          /opt/interop/bin/pip install --no-cache-dir h5py numpy netCDF4 xarray hdf5plugin maturin pytest
+          /opt/interop/bin/pip install --no-cache-dir h5py numpy netCDF4 xarray h5netcdf hdf5plugin maturin pytest
           echo "/opt/interop/bin" >> "$GITHUB_PATH"
       - name: Show interop library versions
         # h5dump's version too: the h5rs dump test requires its exact output

.gitea/workflows/conformance.yml

@@ -40,6 +40,8 @@ jobs:
         run: cargo test --release --manifest-path conformance/probe/Cargo.toml
         env:
           CARGO_TARGET_DIR: conformance/.cache/target
+      - name: Reference-side tests
+        run: /opt/conformance/bin/python conformance/test_ref.py
       - name: Sweep
         # The corpora come from GitHub (pinned commits, conformance/corpus.txt),
         # so this job needs a runner that reaches github.com.

.gitignore

@@ -7,3 +7,6 @@ weights/
 .venv
 __pycache__/
 .pytest_cache/
+
+# Scratch files the heavy tests generate (huge_chunks_interop)
+crates/*/tests/scratch/

BENCHMARKS.md

@@ -30,12 +30,15 @@ target: Criterion stretched it where 5 s could not hold the samples it needed
 > and in memory), Consolidation Efficiency, Ephemeral Tier, Multi-modal Search,
 > the Search and Read harnesses, and the "h5bench-Equivalent I/O Benchmarks"
 > and "Independent Validation: tank" sections. What does not yet meet that bar:
-> the LongMemEval rows that need real embeddings (not re-run here, except the
+> the Consolidation Efficiency 100K cycle row and
-> dated float16 comparison), the Consolidation Efficiency 100K cycle row and
 > memory-reduction part (the 2026-09-24 run was stopped before it produced
 > them), the int8 side of "Quantising the index copy" (not re-run), and the
 > i7-12650H and macOS M3 Max rows under Cross-Platform Notes. That is a
 > known, tracked documentation gap, not a claim that those numbers are wrong.
+> The LongMemEval rows that need real embeddings (vector-only, hybrid, RRF,
+> stemmed hybrid, re-ranking, the weight sweep, the oracle variant) were
+> re-run on 2026-09-27 on tank; see "Re-run with real embeddings" under
+> LongMemEval Results.
 >
 > **Correctness note (2026-08-06).** Being dated and reproducible is necessary but
 > not sufficient — a number can be perfectly reproducible and still measure the
@@ -48,6 +51,31 @@ target: Criterion stretched it where 5 s could not hold the samples it needed
 
 ---
 
+## Current headline numbers
+
+The newest dated measurement of each headline figure, as of 2026-09-28.
+Everything below this section is the dated record behind them; sections whose
+figures a later run replaced are marked *Superseded*. Machine "tank" is an AMD
+Ryzen 7 7800X3D (8C/16T); rows marked idle were run with the 1-minute load
+average below 2.
+
+| Figure | Value | Measured | Command | Details |
+|---|---|---|---|---|
+| Agent memory search, `HDF5Memory::hybrid_search` p50 | 0.49 ms at 10K, 4.69 ms at 100K records | 2026-09-24, tank, `5c8323c` | `cargo run --release -p clawhdf5-bench --bin search_harness -- --full` | [Current: search harness](#current-search-harness-2026-09-24) |
+| LongMemEval `longmemeval_s` (full haystack), default hybrid 0.4/0.6, turn-level retrieval Hit@5 (not QA accuracy) | 81.4% | 2026-09-27, tank, search code of `7a8fae0` | `longmemeval_bench … --embeddings weights/all-minilm-l6-v2` | [Re-run with real embeddings](#re-run-with-real-embeddings-2026-09-27-tank), [Fusion method](#fusion-method--weighted-vs-rrf-full-haystack-n500) |
+| Loaded store memory, 100K × 384 | 399 MiB (2.72x raw) with the `f32` index; 256 MiB (1.74x) with the int8 index (int8 side not re-run since it was first measured) | `f32`: 2026-09-24, tank, `5c8323c`; int8: 2026-09-19 (`c0a9206`), machine not recorded | `search_harness -- --footprint --full [--int8]` | [Memory footprint](#memory-footprint), [Quantising the index copy](#quantising-the-index-copy-quantized_index) |
+| int8 index vs `f32` index, QPS at equal recall | 1.63x (x86-64 AVX2), 1.18x (Raspberry Pi 5, `SDOT`) | x86: 2026-09-20 (`dea02f5`), machine not recorded; Pi 5: 2026-09-21 (`114a2df`); not re-checked against the 2026-09-24 `f32` figure | `search_harness -- --full` | [Quantising the index copy](#quantising-the-index-copy-quantized_index), [On ARM](#on-arm-raspberry-pi-5-cortex-a76) |
+| `float16` store file size, 100K × 384 | 80.8 MiB vs 154.0 MiB `f32` (48% smaller) | 2026-09-23, tank | `search_harness -- --float16-study --full` | [float16 embedding storage](#float16-embedding-storage-memoryconfigfloat16) |
+| Full reads of chunked deflate data, 16 threads on one `File` | 4944 MB/s, 1.58x 16 h5py processes (noisy run: compare ratios, not MB/s) | 2026-09-26, tank, `c5334b1` | `concurrent_read` + `concurrent_read_h5py.py` | [Results after in-place chunk decoding](#results-after-in-place-chunk-decoding-2026-09-26-tank-c5334b1) |
+| Same, clawhdf5 only, against the build before range-read M2/M3 | 8525 MB/s vs 6258 (+36%); contiguous and metadata reads at parity | 2026-09-27, tank (idle), `7a8fae0` vs `8f59b2e` | `concurrent_read --decode-threads 1 --reps 3` | [Local metadata and data reads after range-read M2/M3](#local-metadata-and-data-reads-after-range-read-m2m3-2026-09-27-tank) |
+| `ObjectHeader::parse` (401 headers) | 23.5–23.6 µs, 1.0–2.6% below `8f59b2e` | 2026-09-27, tank (idle), `96086ad` | `cargo bench -p clawhdf5 --bench local_metadata_bench` | [`ObjectHeader::parse` back at 8f59b2e's speed](#objectheaderparse-back-at-8f59b2es-speed-2026-09-27-tank) |
+| Selection reads, 64 MB chunked + deflate `f64` | full 63.2 ms; one 64 × 64 window 0.18 ms | 2026-09-24, tank, `5c8323c` | `cargo run --release -p clawhdf5-bench --bin read_harness` | [Current: read harness](#current-read-harness-2026-09-24) |
+| Deflate backend, zlib-rs (default) vs zlib-ng | within 6% on every HDF5 read/write path | 2026-09-23, tank | `cargo bench -p clawhdf5-filters --bench deflate_bench` (and the two commands with it) | [Deflate backend](#deflate-backend-zlib-rs-vs-zlib-ng) |
+| vs libhdf5 1.14.6: chunked deflate-6 write 512×512 / 128 attributes / 64 groups | 35x (1.46 vs 51.4 ms, pure-Rust deflate) / 10.3x / 10.6x | write 2026-09-23, tank; attributes and groups 2026-08-03, tank | `cargo bench -p clawhdf5-bench --bench h5bench_write --features libhdf5-compare -- '^write_2d_chunked/'`; `cargo bench -p clawhdf5-bench --features libhdf5-compare` | [Deflate backend](#deflate-backend-zlib-rs-vs-zlib-ng), [Independent Validation: tank](#independent-validation-tank-ryzen-7-7800x3d-2026-08-03) |
+| Signed checkpoints | about 20% of a checkpoint (598 vs 495 ms at 100K) | 2026-09-25, tank | `search_harness -- --signing-study --full` | [Signed checkpoints](#signed-checkpoints) |
+
+---
+
 ## Memory footprint
 
 `cargo run --release -p clawhdf5-bench --bin search_harness -- --footprint --full`,
@@ -61,6 +89,10 @@ change at all. Measured that way a store holding the corpus twice and one
 holding it once came out *identical* (1.00x both), which is how the first
 attempt at this measurement went.
 
+> *Superseded* by the current figures below (2026-09-24): this table is the
+> record of the double-copy fix (commit 2e7e045, undated); the store measured
+> 2.72x, not 2.43x, by the time the int8 index landed.
+
 | N | vectors (raw) | reopened, before | reopened, after |
 |---:|---:|---:|---:|
 | 1 000 | 1 MiB | 5 MiB (3.41x) | 4 MiB (2.39x) |
@@ -325,6 +357,19 @@ which of two gold sessions ranks first, out of ~320. Those flips show the
 half-precision path was in effect; they do not change a single hit. The f32
 run reproduces the published hybrid numbers exactly.
 
+**Re-checked 2026-09-27** (tank, commit 7a8fae0, the same pair of runs on
+`longmemeval_s_cleaned.json`, which is the same file; the machine was not
+idle, which does not affect recall): the result is the same. The table above
+reproduced exactly. Every Hit@k and MRR of the eight modes matched between f32
+and float16 at both levels, with two exceptions: RRF's session MRR (0.9253 vs
+0.9254) and two per-type session MRRs in the fourth decimal. Three modes
+differed by one question in the recency count. That re-run also corrects the
+sentence above: two f32 runs on the same day differed by one question in
+recency as well, so those flips are run-to-run variation and do not show that
+the half-precision path was in effect. `--float16` is what shows that: the
+harness prints "Stores use MemoryConfig::float16" and `MemoryConfig::float16`
+is set on every store.
+
 ### Opening a store (`read_from_disk`)
 
 `HDF5Memory::open` memory-mapped the file, copied the whole mapping into a
@@ -363,6 +408,9 @@ point: does a selection cost what the *selection* costs?
 
 ### Baseline (v2.4.0): every selection decodes the whole dataset
 
+> *Superseded* by [Current: read harness](#current-read-harness-2026-09-24)
+> (2026-09-24). Kept as the before picture.
+
 4096 x 2048 f64 (64 MB per dataset), chunks 256 x 256, file 129 MB
 
 | layout | read | selected | time ms | MB/s of selection | vs full read |
@@ -388,6 +436,9 @@ point: does a selection cost what the *selection* costs?
 
 ### After: partial reads
 
+> *Superseded* by [Current: read harness](#current-read-harness-2026-09-24)
+> (2026-09-24).
+
 Only the rows of a contiguous dataset, or the chunks, that overlap the
 selection's bounding box are read/decoded. A 64 x 64 window of the compressed
 dataset: **105 -> 0.39 ms**; one row: **106 -> 2.7 ms**; one column:
@@ -419,6 +470,9 @@ because the machine's speed drifted; compare the *vs full read* column.)
 
 ### After: parallel cached decode, fewer copies (full reads)
 
+> *Superseded* by [Current: read harness](#current-read-harness-2026-09-24)
+> (2026-09-24).
+
 Full-read times, old and new binaries run alternately at the same moment (this
 machine's absolute speed drifts over a long session, so only same-moment
 comparisons mean anything):
@@ -482,10 +536,265 @@ The rows and columns of the uncompressed layouts are within 20% (chunked
 column 0.45 -> 0.49 ms, contiguous column 2.55 -> 2.61 ms). This run does not
 explain the slower windows.
 
-## Concurrent reads
+### HDF5 1.8 format: version-1 B-tree chunk indexes (2026-09-28, tank, loaded)
+
+**Superseded** by the idle re-run below; kept as the record the default
+was first decided on.
+
+Measured 2026-09-28 on tank (AMD Ryzen 7 7800X3D), branch `feat/libver-v18`
+at `3c61635`, to decide whether the writer's default should become the
+HDF5 1.8 format (`libver_bounds(LibVer::V18, LibVer::V18)`: version-1 B-tree
+chunk indexes) instead of the 1.10 format (Fixed Array indexes for these
+datasets). **Not an idle machine:** two other agents were building; the
+1-minute load average was 7.0 to 7.6 throughout (the rule is below 2), so
+treat differences under about 20% as noise. Default and `--v18` runs
+alternated, three of each per chunk size; medians of the three.
+
+> **Run:** `cargo run --release -p clawhdf5-bench --bin read_harness -- --chunk N [--v18]`
+> with N = 256 (128 chunks per dataset) and N = 32 (8192 chunks per dataset)
+
+Write (the whole 3-dataset file) and file size:
+
+| chunks | 1.10 write ms | 1.8 write ms | 1.10 file bytes | 1.8 file bytes | size |
+|---|---:|---:|---:|---:|---:|
+| 256 x 256 | 467-500 | 467-483 | 134 916 080 | 134 933 848 | +0.013% |
+| 32 x 32 | 490-495 | 489-506 | 138 879 336 | 139 465 112 | +0.42% |
+
+Reads (chunked datasets; the contiguous one does not change), ms. The
+window labels are the harness's, which count 256 x 256 chunks: with 32 x 32
+chunks the 64 x 64 window covers 9 chunks and the 512 x 512 one 289.
+
+| chunks | layout | read | 1.10 | 1.8 | 1.8 / 1.10 |
+|---|---|---|---:|---:|---:|
+| 256 | chunked + deflate | full (first) | 6.70 | 7.30 | 1.09x |
+| 256 | chunked + deflate | full (repeat) | 4.80 | 3.80 | 0.79x |
+| 256 | chunked + deflate | 64 x 64 window (1 chunk) | 0.15 | 0.16 | 1.07x |
+| 256 | chunked + deflate | 512 x 512 window (4-9 chunks) | 2.40 | 1.25 | 0.52x |
+| 256 | chunked + deflate | one row | 0.95 | 0.95 | 1.00x |
+| 256 | chunked + deflate | one column | 1.93 | 1.94 | 1.01x |
+| 256 | chunked | full (first) | 11.30 | 11.80 | 1.04x |
+| 256 | chunked | full (repeat) | 6.20 | 5.60 | 0.90x |
+| 256 | chunked | 64 x 64 window (1 chunk) | 0.04 | 0.04 | 1.00x |
+| 256 | chunked | 512 x 512 window (4-9 chunks) | 1.50 | 0.37 | 0.25x |
+| 256 | chunked | one row | 0.05 | 0.05 | 1.00x |
+| 256 | chunked | one column | 0.44 | 0.45 | 1.02x |
+| 32 | chunked + deflate | full (first) | 11.50 | 12.20 | 1.06x |
+| 32 | chunked + deflate | full (repeat) | 9.30 | 9.30 | 1.00x |
+| 32 | chunked + deflate | 64 x 64 window (1 chunk) | 0.45 | 0.89 | 1.98x |
+| 32 | chunked + deflate | 512 x 512 window (4-9 chunks) | 1.97 | 2.41 | 1.22x |
+| 32 | chunked + deflate | one row | 0.69 | 1.13 | 1.64x |
+| 32 | chunked + deflate | one column | 1.26 | 1.70 | 1.35x |
+| 32 | chunked | full (first) | 11.80 | 13.00 | 1.10x |
+| 32 | chunked | full (repeat) | 6.30 | 6.20 | 0.98x |
+| 32 | chunked | 64 x 64 window (1 chunk) | 0.36 | 0.83 | 2.31x |
+| 32 | chunked | 512 x 512 window (4-9 chunks) | 0.70 | 1.16 | 1.66x |
+| 32 | chunked | one row | 0.38 | 0.84 | 2.21x |
+| 32 | chunked | one column | 1.05 | 1.21 | 1.15x |
+
+With 128 chunks per dataset the two formats read and write alike (the 512 x
+512 windows' 0.25x and 0.52x are not explained by the index and are likely
+the load). With 8192 chunks, full reads stay within 10%, but a selection on
+a freshly opened file costs about 0.4 to 0.5 ms more through the version-1
+B-tree (1.2x to 2.3x). Each timed selection opens the file anew, so the
+likely cause (not profiled) is walking the B-tree's nodes (2.6 KB each,
+about 150 per dataset here) against a Fixed Array's few blocks. Writing costs the
+same; files grow by about 36 bytes per chunk. The default therefore stays
+the 1.10 format; the 1.8 format is opt-in.
+
+### HDF5 1.8 format: version-1 B-tree chunk indexes, idle re-run (2026-09-28, tank)
+
+Measured 2026-09-28 on tank (AMD Ryzen 7 7800X3D), idle: 1-minute load
+average 1.66 to 1.84 at the start of each run. Stacked branch
+`feat/huge-chunks` at `b55768f` (contains `feat/libver-v18`). Default and
+`--v18` runs alternated, five of each per chunk size; median (min-max), ms.
+
+> **Run:** `cargo run --release -p clawhdf5-bench --bin read_harness -- --chunk N [--v18]`
+> with N = 256 (128 chunks per dataset) and N = 32 (8192 chunks per dataset)
+
+| chunks | layout | read | 1.10 | 1.8 | 1.8 / 1.10 |
+|---|---|---|---:|---:|---:|
+| 256 | chunked + deflate | full (first) | 5.9 (5.5-6.2) | 6.2 (6.0-6.7) | 1.05x |
+| 256 | chunked + deflate | full (repeat) | 4.4 (4.1-4.9) | 4.3 (4.2-4.4) | 0.98x |
+| 256 | chunked + deflate | 64 x 64 window (1 chunk) | 0.15 (0.15-0.17) | 0.16 (0.15-0.16) | 1.07x |
+| 256 | chunked + deflate | 512 x 512 window (4-9 chunks) | 1.23 (1.23-1.36) | 1.22 (1.22-1.25) | 0.99x |
+| 256 | chunked + deflate | one row | 0.95 (0.94-1.04) | 0.94 (0.93-0.98) | 0.99x |
+| 256 | chunked + deflate | one column | 1.92 (1.91-2.12) | 1.90 (1.90-1.91) | 0.99x |
+| 256 | chunked | full (first) | 11.1 (10.7-11.6) | 10.9 (10.6-11.9) | 0.98x |
+| 256 | chunked | full (repeat) | 5.1 (4.9-5.2) | 5.2 (4.7-5.6) | 1.02x |
+| 256 | chunked | 64 x 64 window (1 chunk) | 0.03 (0.03-0.03) | 0.04 (0.04-0.04) | 1.33x |
+| 256 | chunked | 512 x 512 window (4-9 chunks) | 0.36 (0.36-0.38) | 0.37 (0.36-0.38) | 1.03x |
+| 256 | chunked | one row | 0.05 (0.04-0.05) | 0.05 (0.05-0.05) | 1.00x |
+| 256 | chunked | one column | 0.45 (0.43-0.45) | 0.44 (0.44-0.45) | 0.98x |
+| 32 | chunked + deflate | full (first) | 11.7 (11.2-12.4) | 12.5 (11.8-13.2) | 1.07x |
+| 32 | chunked + deflate | full (repeat) | 9.5 (9.3-9.8) | 9.4 (9.3-11.2) | 0.99x |
+| 32 | chunked + deflate | 64 x 64 window (1 chunk) | 0.46 (0.45-0.49) | 0.89 (0.89-0.90) | 1.93x |
+| 32 | chunked + deflate | 512 x 512 window (4-9 chunks) | 1.95 (1.94-2.04) | 2.40 (2.39-2.51) | 1.23x |
+| 32 | chunked + deflate | one row | 0.69 (0.67-0.72) | 1.15 (1.13-1.18) | 1.67x |
+| 32 | chunked + deflate | one column | 1.26 (1.24-1.28) | 1.72 (1.69-1.73) | 1.37x |
+| 32 | chunked | full (first) | 12.5 (11.9-12.7) | 12.6 (12.5-13.4) | 1.01x |
+| 32 | chunked | full (repeat) | 5.9 (5.8-6.0) | 6.2 (6.0-6.4) | 1.05x |
+| 32 | chunked | 64 x 64 window (1 chunk) | 0.36 (0.35-0.36) | 0.84 (0.83-0.86) | 2.33x |
+| 32 | chunked | 512 x 512 window (4-9 chunks) | 0.70 (0.69-0.74) | 1.17 (1.16-1.17) | 1.67x |
+| 32 | chunked | one row | 0.38 (0.38-0.41) | 0.85 (0.84-0.86) | 2.24x |
+| 32 | chunked | one column | 1.03 (1.02-1.21) | 1.22 (1.20-1.25) | 1.18x |
+
+Writes: 377 vs 374 ms (128 chunks), 399 vs 409 ms (8192 chunks); file
+bytes as in the loaded run. The loaded run's 0.25x and 0.52x windows at 128
+chunks were the load: idle, every 128-chunk read is within 7% except the
+0.03 ms single-chunk window (one timer tick). The 8192-chunk result stands:
+a selection on a freshly opened file costs 0.2 to 0.5 ms more through the
+version-1 B-tree (1.2x to 2.3x), full reads are within 7%. The default stays
+the 1.10 format.
+
+## Local file speed after range reads
+
+### `ObjectHeader::parse` back at 8f59b2e's speed (2026-09-27, tank)
+
+The remaining 4% (below) was the call to the version-1 message loop, which
+`4313917` kept out of line with `#[inline(never)]`. Found with A/B builds
+changing one piece at a time (perf is not available: `perf_event_paranoid`
+4): `#[inline]` on `parse_v1_messages` alone brought
+`object_header_parse_x401` from about 24.5–24.9 µs to 23.6–24.0 µs against
+8f59b2e's 23.6–24.1 µs (short 4-second rounds); no attribute measured like
+`#[inline(never)]`;
+creating the chunk list only when a continuation is found measured no
+faster on top and was not kept.
+
+Same method as below: `8f59b2e` built in its own worktree and target
+directory, separate binaries alternating, `taskset -c 5
+local_metadata_bench --bench --warm-up-time 3 --measurement-time 10`, every
+binary started with the 1-minute load average below 2 (0.19–1.86) and no
+`rustc` running. Candidate: `96086ad` (this change). Median (range) of 3
+rounds; run 2 also alternated `main` `425585e`.
+
+| function | 8f59b2e | 425585e (main) | 96086ad | vs 8f59b2e |
+|---|---:|---:|---:|---:|
+| run 1: `object_header_parse_x401` | 23.81 µs (23.76–24.00) | | 23.57 µs (23.23–23.89) | **−1.0%** |
+| run 1: `snod_parse_all` | 1.840 µs (1.837–1.854) | | 1.839 µs (1.837–1.873) | 0.0% |
+| run 1: `btree_v1_walk` | 343 ns (338–349) | | 352 ns (344–363) | +2.7% |
+| run 1: `facade_list_400_groups` | 8.03 ms (8.01–8.10) | | 8.05 ms (8.01–8.15) | +0.2% |
+| run 2: `object_header_parse_x401` | 24.15 µs (23.74–24.41) | 24.93 µs (24.72–25.05) | 23.52 µs (23.34–23.68) | **−2.6%** |
+| run 2: `snod_parse_all` | 1.843 µs (1.830–1.847) | 1.856 µs (1.850–1.865) | 1.861 µs (1.858–1.869) | +1.0% |
+| run 2: `btree_v1_walk` | 346 ns (339–366) | 356 ns (347–356) | 357 ns (350–359) | +3.2% |
+| run 2: `facade_list_400_groups` | 8.09 ms (8.02–8.10) | 8.12 ms (7.97–8.19) | 8.08 ms (7.98–8.12) | −0.1% |
+
+- `ObjectHeader::parse` is at or below 8f59b2e (−1.0%, −2.6%) and 5.6%
+  faster than `main` in the same run.
+- `btree_v1_walk` (one walk of a 350 ns B-tree) is 3% above 8f59b2e in
+  both runs, with overlapping ranges, and is the same on `main` (+0.2%
+  between `main` and this change): not from this change. The walk's code
+  changed in `e553153` (after a failed child the siblings are only read,
+  so the error returns after them; the fixture never takes that path, but
+  the loop carries the extra state); left as is.
+- `snod_parse_all` and the facade listing are within noise.
+
+### Local metadata and data reads after range-read M2/M3 (2026-09-27, tank)
+
+> The `object_header_parse_x401` row (+4.2%) is *superseded* by
+> [`ObjectHeader::parse` back at 8f59b2e's speed](#objectheaderparse-back-at-8f59b2es-speed-2026-09-27-tank)
+> (2026-09-27, `96086ad`); the other rows are current.
+
+`main` just before range-read M2/M3 (`8f59b2e`, PR #17) against `main`
+`7a8fae0` (PRs #18 and #19), each built in its own worktree and run as
+separate binaries, alternating base and candidate. Machine: tank (AMD Ryzen
+7 7800X3D, 16 threads). **Idle:** every round started with the 1-minute load
+average below 2 (1.05–1.98; `target/ab-results2/load.log`). Criterion:
+`taskset -c 5 local_metadata_bench --bench --warm-up-time 3
+--measurement-time 10`, 3 rounds each. Reads: `concurrent_read --dir
+~/.cache/concurrent-read --decode-threads 1 --reps 3`, 3 rounds each.
+Median (range) over the rounds.
+
+`local_metadata_bench` (the 400-group v1 fixture):
+
+| function | 8f59b2e | 7a8fae0 | change |
+|---|---:|---:|---:|
+| `object_header_parse_x401` | 23.86 µs (23.79–23.96) | 24.86 µs (24.69–24.98) | **+4.2%** |
+| `snod_parse_all` | 1.842 µs (1.835–1.847) | 1.833 µs (1.833–1.854) | −0.5% |
+| `btree_v1_walk` | 344 ns (338–346) | 348 ns (337–356) | +1.1% |
+| `facade_list_400_groups` | 8.24 ms (8.04–8.28) | 8.10 ms (8.04–8.13) | −1.7% |
+
+`concurrent_read`, MB/s (64 datasets of 64 MiB `f32`; deflate chunks
+256 x 256, level 4):
+
+| layout | mode | threads | 8f59b2e | 7a8fae0 | change |
+|---|---|---:|---:|---:|---:|
+| deflate | distinct | 1 | 891 (880–892) | 907 (906–908) | +1.7% |
+| deflate | distinct | 2 | 1692 (1680–1699) | 1777 (1767–1779) | +5.0% |
+| deflate | distinct | 4 | 3102 (3099–3102) | 3348 (3347–3350) | +8.0% |
+| deflate | distinct | 8 | 5306 (5168–5345) | 6240 (6226–6248) | +17.6% |
+| deflate | distinct | 16 | 6258 (6109–6442) | 8525 (8513–8561) | **+36.2%** |
+| deflate | same | 1 | 234 (234–235) | 233 (233–234) | −0.5% |
+| deflate | same | 16 | 2443 (2038–2444) | 2468 (2424–2473) | +1.0% |
+| contiguous | distinct | 1 | 13477 (12949–13874) | 13302 (13287–13578) | −1.3% |
+| contiguous | distinct | 16 | 12501 (12484–12530) | 12566 (12449–12567) | +0.5% |
+| contiguous | same | 1 | 29866 (28832–30207) | 29364 (28838–29780) | −1.7% |
+| contiguous | same | 16 | 163415 (161097–229146) | 233280 (159377–238440) | (noise) |
+
+What this shows:
+- **Local metadata reads are at parity or faster.** Listing the 400-group
+  file through the facade is 1.7% faster than before M2/M3; the +7–10%
+  listing regression found while merging #18 is gone.
+- **`ObjectHeader::parse` alone is 4.2% slower** (about 2.5 ns per header;
+  the base and candidate ranges do not overlap). It is the cost of reading
+  continuation chunks from a bounded queue (the fix for unbounded reads on
+  crafted headers) and does not show in the listing. (Fixed later the
+  same day; see the section above and `docs/known-issues.md`.)
+- **Full reads of deflate data got faster** after #18 (in-place chunk
+  decoding into the typed output and per-thread scratch buffers): +1.7% on
+  one thread, +36% at 16.
+- Single-thread contiguous hyperslabs are within noise (−1.7%, overlapping
+  ranges). The multi-thread `contiguous same` rows read one 64 MiB dataset
+  out of the CPU caches and swing widely between rounds of the same build.
+
+An earlier run the same day at load 2.3–3.3 (two orphaned h5py processes,
+since stopped, each using a core) reported that single-thread contiguous
+hyperslab row as −5.6%; the idle rerun above does not reproduce it.
+
+### Results after in-place chunk decoding (2026-09-26, tank, `c5334b1`)
+
+Same machine, files and commands, re-run after chunked reads started
+decoding into reusable per-thread buffers straight into the (typed) output,
+with the calling thread decoding alongside the pool. Load average 1.78 at
+the start; it rose to 6-9 during the runs (the clawhdf5 runs' own threads,
+and it stayed around 5-6 through the h5py runs, so something else was
+active). **This run was noisier than the previous one: h5py's own contiguous
+figures are about 40% lower than in the run below, and ours dropped
+similarly, so compare ratios within a run rather than MB/s across runs.**
+h5py was re-run in the same session.
+
+Each read decoding on its calling thread (`--decode-threads 1`, like h5py):
+
+| layout | mode | threads | clawhdf5 MB/s (eff) | h5py threads MB/s (eff) | h5py processes MB/s (eff) | vs h5py processes |
+|---|---|---:|---:|---:|---:|---:|
+| deflate | distinct | 1 | 670 (1.00) | 410 (1.00) | 397 (1.00) | 1.69x |
+| deflate | distinct | 4 | 2434 (0.91) | 406 (0.25) | 1470 (0.93) | 1.66x |
+| deflate | distinct | 8 | 3749 (0.70) | 406 (0.12) | 2398 (0.76) | 1.56x |
+| deflate | distinct | 16 | 4944 (0.46) | 390 (0.06) | 3135 (0.49) | 1.58x |
+| deflate | same | 1 | 211 (1.00) | 125 (1.00) | 124 (1.00) | 1.70x |
+| deflate | same | 16 | 1835 (0.54) | 122 (0.06) | 961 (0.48) | 1.91x |
+| contiguous | distinct | 1 | 6718 (1.00) | 5545 (1.00) | 5200 (1.00) | 1.29x |
+| contiguous | distinct | 16 | 11035 (0.10) | 4950 (0.06) | 10558 (0.13) | 1.05x |
+| contiguous | same | 1 | 14483 (1.00) | 2593 (1.00) | 2737 (1.00) | 5.29x |
+| contiguous | same | 16 | 132175 (0.57) | 2224 (0.05) | 14809 (0.34) | 8.93x |
+
+With the default rayon pool, deflate `distinct` reads 6143 MB/s from a single
+thread (15x h5py's 410 on one call) and 4556 MB/s at 16 threads (1.45x h5py
+processes); the other rows are within the noise of the table above.
+
+What changed: full reads of chunked datasets were 0.69x-0.76x of h5py
+processes at 16 threads in the run below, and are 1.58x here; with one
+thread they were 1.44x and are 1.69x. Minor page faults for the 16-thread
+run fell from about 4.6M to 0.2M (`/usr/bin/time -v`, provisional, loaded
+machine). clawhdf5 now reads faster than 16 h5py processes in every row of
+this benchmark except contiguous full reads at 16 threads, where both
+saturate memory bandwidth (1.05x).
 
 ### Results after the read fixes (2026-09-26, tank, `408f69e`)
 
+> *Superseded* by [Results after in-place chunk decoding](#results-after-in-place-chunk-decoding-2026-09-26-tank-c5334b1)
+> (2026-09-26, `c5334b1`), which closed the 16-thread gap listed at the end
+> of this section.
+
 Same machine, files and commands as the first run below, re-run on an idle
 tank (load average 1.60 at the start; the 1-minute figure rose to about 5
 during the clawhdf5 runs, mostly their own threads) after two fixes:
@@ -526,11 +835,17 @@ Read with care:
 - At 16 threads every tool dropped in this run (h5py threads on contiguous
   data from 8002 to 2285 MB/s, processes from 12846 to 6942), so the
   16-thread rows are noisier than the others.
-- Still behind: full reads of chunked data at 16 threads (0.69x-0.76x h5py
+- Still behind at this commit: full reads of chunked data at 16 threads
-  processes). See `docs/known-issues.md`.
+  (0.69x-0.76x h5py processes); fixed by `c5334b1` (above), recorded as
+  fixed in `docs/known-issues.md`.
 
 ### First run, before the read fixes (2026-09-26, tank, `91644d8`)
 
+> *Superseded* results: the tables and "What this shows" are the before
+> picture for [Results after in-place chunk decoding](#results-after-in-place-chunk-decoding-2026-09-26-tank-c5334b1)
+> (2026-09-26). The workload description and the **Run** box below are
+> still how every `concurrent_read` figure in this file is produced.
+
 Measured on tank (AMD Ryzen 7 7800X3D, 8 cores / 16 threads, 61 GiB, Linux
 7.0) at commit `91644d8`, load average 1.84 when the run started (the
 1-minute figure rose to 3.7 during the runs; that is mostly the benchmark's
@@ -567,8 +882,8 @@ What this shows:
 - **clawhdf5 threads on one `File` do, for hyperslab reads of compressed
   data:** 1244 MB/s at 16 threads, 9.7x h5py threads and 0.89x h5py
   processes, without a process pool.
-- **Where clawhdf5 is behind** (open performance bugs, see
+- **Where clawhdf5 was behind** at `91644d8` (both since fixed; see
-  `docs/known-issues.md`):
+  `docs/known-issues.md`, "Concurrent and contiguous read performance"):
   - *Full reads of chunked datasets stop scaling at about 4 threads*
     (about 880 MB/s) while h5py processes reach 4424 MB/s. Hyperslab
     reads, which bypass the `File`'s chunk cache, keep scaling, so the
@@ -658,6 +973,11 @@ Other flags (both harnesses): `--threads`, `--reps`, `--slab`, `--slabs`,
 
 ## Search harness baseline (v2.3.0)
 
+> *Historical.* This baseline and the "After: …" subsections that follow
+> record each step of the search work; they are *superseded* by
+> [Current: search harness](#current-search-harness-2026-09-24) (2026-09-24),
+> the last subsection of this part.
+
 Produced by `cargo run --release -p clawhdf5-bench --bin search_harness -- --full`
 on deterministic **clustered** synthetic data (384-dim, unit-normalised; points =
 cluster centre + noise — uniform random vectors are nearly equidistant in high
@@ -720,10 +1040,11 @@ build: 9752.6 ms (10254 vectors/s) · exact scan: 40 QPS, p50 24648 µs
 | 1000 | 11 | 3.9 | 0.9 | 68.1 | 5.48 | 5.57 | 182.5 |
 | 10000 | 114 | 32.2 | 10.9 | 845.0 | 48.56 | 78.65 | 19.8 |
 | 100000 | 1486 | 713.0 | 354.5 | 10486.5 | 883.51 | 975.23 | 1.1 |
-wrote /tmp/claude-1000/-home-osobh-projects-clawhdf5/422f755e-dd25-4c35-8613-5439087e3aaa/scratchpad/baseline_full.json
 
 ### After: HNSW neighbour-selection heuristic
 
+> *Superseded* by [Current: search harness](#current-search-harness-2026-09-24) (2026-09-24).
+
 Same harness, same data, after replacing closest-M neighbour selection with the
 HNSW paper's diversity heuristic (Algorithm 4, keeping pruned connections) for
 both new links and back-link pruning. Recall@10 at `ef = 64`: **0.87 → 1.00**
@@ -769,6 +1090,8 @@ build: 36472.8 ms (2742 vectors/s) · exact scan: 40 QPS, p50 24644 µs
 
 ### After: persistent keyword index, no store rewrite per query
 
+> *Superseded* by [Current: search harness](#current-search-harness-2026-09-24) (2026-09-24).
+
 `hybrid_search` used to rebuild the BM25 index from scratch (re-tokenising every
 record) and rewrite the whole `.h5` file on **every query**. The index is now
 kept for the life of the store and updated incrementally, and activation boosts
@@ -789,6 +1112,8 @@ index removes that.
 
 ### After: vector index persisted with the checkpoint
 
+> *Superseded* by [Current: search harness](#current-search-harness-2026-09-24) (2026-09-24).
+
 The HNSW graph (not the vectors, which the store already holds) is saved to
 `<store>.h5.ann` at each checkpoint and reloaded by `open()`, tied to that
 checkpoint by a generation id. The index is now built once per store (the *cold
@@ -806,6 +1131,8 @@ index incrementally.
 
 ### After: unit-vector dot product, reusable visited set
 
+> *Superseded* by [Current: search harness](#current-search-harness-2026-09-24) (2026-09-24).
+
 Cosine distance recomputed both vector norms on every evaluation; the index now
 stores unit vectors and uses a plain dot product. The per-call `HashSet` of
 visited nodes became a reusable epoch-stamped array. Recall is unchanged.
@@ -851,6 +1178,8 @@ build: 21084.6 ms (4743 vectors/s) · exact scan: 39 QPS, p50 24739 µs
 
 ### After: unranked keyword scores, top-k merge (rankings unchanged)
 
+> *Superseded* by [Current: search harness](#current-search-harness-2026-09-24) (2026-09-24).
+
 A fusion study (`search_harness --fusion-study`) showed that capping the
 keyword candidate pool is **not** a safe optimisation: against the current
 full-corpus normalisation the final top-10 overlap is only 0.83-0.92 and the
@@ -873,6 +1202,8 @@ results.
 
 ### After: batched bulk build (optionally parallel); deletions handled in search
 
+> *Superseded* by [Current: search harness](#current-search-harness-2026-09-24) (2026-09-24).
+
 Profiling showed **90% of a build's distance evaluations are in back-link
 pruning**. The bulk build now inserts in batches: plan each node's neighbours
 against the graph as it stood at the start of the batch, link, then prune every
@@ -1371,10 +1702,75 @@ turn-level row plus its session Hit@1 in the tokenizer table. The run also
 produced figures this document does not publish (stemmed session Hit@5,
 Hit@10 and MRR, and per-type Hit@5/Hit@10/MRR for both modes), so there was
 nothing to compare them with. Rows that need real embeddings (vector-only,
-hybrid, RRF, re-ranking, the weight sweep) were not re-run.
+hybrid, RRF, re-ranking, the weight sweep) were not re-run then; they were on
+2026-09-27 (next section).
+
+### Re-run with real embeddings (2026-09-27, tank)
+
+Every recall row in this section that needs real embeddings was measured
+again on 2026-09-27 on tank (AMD Ryzen 7 7800X3D, 16 threads; MiniLM
+embeddings on an RTX 5060 Ti, retrieval on the CPU), with the search code of
+commit 7a8fae0. Six runs:
+
+```bash
+cargo build --release -p clawhdf5-bench --bin longmemeval_bench --features embeddings-cuda
+B=target/release/longmemeval_bench W=weights/all-minilm-l6-v2
+$B benchmarks/longmemeval/longmemeval_oracle.json     --embeddings $W
+$B benchmarks/longmemeval/longmemeval_s_cleaned.json  --embeddings $W
+$B benchmarks/longmemeval/longmemeval_s_cleaned.json  --embeddings $W --float16
+$B benchmarks/longmemeval/longmemeval_oracle.json     --embeddings $W --sweep
+$B benchmarks/longmemeval/longmemeval_s_cleaned.json  --embeddings $W --sweep
+$B benchmarks/longmemeval/longmemeval_s_cleaned.json  --embeddings $W --rerank-sweep
+```
+
+(`longmemeval_s.json`, used by the commands elsewhere in this section, is the
+same file as `longmemeval_s_cleaned.json`.)
+
+**The machine was not idle.** The 1-minute load never fell below 2 in a
+2-hour wait, because two stray test processes were each holding a core; it
+was 2.1–2.7 when each run started and up to 8.4 while the runs were going.
+Recall does not depend on load. Latency does, so no latency figure in this
+file was updated from these runs.
+
+**What reproduced exactly:** every published full-haystack Hit@1/5/10 and MRR
+at turn and session level for BM25, vector-only, hybrid 0.4/0.6, both stemmed
+modes, and every re-ranking row; RRF except as below; the float16 table; 4 of
+the 11 weight-sweep rows (0.0, 0.1, 0.3 and 0.9); and the oracle vector-only
+figure. The headline,
+hybrid 0.4/0.6 turn Hit@5 **81.4%**, is unchanged.
+
+**What changed.** Old values are kept here; the tables below show the new ones.
+
+| Figure | Published | 2026-09-27 | Why |
+|---|---:|---:|---|
+| Oracle, hybrid turn Hit@5 | 85.2% (2026-08-07, c913cd1) | **86.8%** | Weights. 85.2% was measured at 0.7/0.3, the default then. The default has been 0.4/0.6 since 29baabb. Today's oracle sweep gives 85.2% at 0.7/0.3 and 86.8% at 0.4/0.6. |
+| Oracle, BM25-only turn Hit@5 with embeddings | 84.2% (2026-08-07) | 84.4% | Now equal to the zero-embedding figure, as it already was on the full haystack. At c913cd1, tied candidates were ordered by `HashMap` iteration. Since 3ed0489 (2026-09-19) they are broken by index. 3ed0489 is the likely cause; it was not bisected. |
+| Ablation / sweep, hybrid 0.7/0.3, turn | 44.4% / 79.2% / 86.0% / 0.5868 | 44.2% / 79.2% / 85.8% / 0.5856 | The same tie-breaking change. 29baabb's re-run on 2026-09-19, made after 3ed0489 that same day, already had 44.2% / 85.8% / 0.5856. |
+| Ablation, hybrid 0.7/0.3, session | 88.2% / 95.8% / 97.8% / 0.9158 | 88.0% / 95.8% / 97.6% / 0.9146 | Same. |
+| Ablation / sweep, vector-only turn MRR | 0.5027 | 0.5031 | Same. The fusion and float16 tables already had 0.5031. |
+| Sweep 0.2/0.8, turn | 53.6% / 78.2% / 0.6440 | 53.4% / 78.0% / 0.6429 | Same (the sweep was measured 2026-08-07, 1537a94). |
+| Sweep 0.4/0.6, 0.5/0.5, 0.8/0.2 turn MRR | 0.6429 / 0.6234 / 0.5571 | 0.6430 / 0.6232 / 0.5574 | Same. |
+| Sweep 0.6/0.4 | Hit@5 79.8%, MRR 0.6069, session Hit@5 96.6% | 79.6%, 0.6067, 96.4% | Same. |
+| RRF turn MRR | 0.5967 (2026-09-19, aa92fef) | 0.5969 | Not explained. It may be a later search-path change, such as the int8 index becoming the default in 8b85d93. It may also be the run-to-run variation described next. It was not bisected. |
+
+**Not exactly deterministic.** Between two f32 runs today, the recency count
+(the share of `knowledge-update` questions where the newest gold session
+ranks first) differed by one question in two modes: hybrid 0.4/0.6 was
+144/320 in one run and 145/320 in the other, and re-rank with a 1-day
+half-life was 165/319 and 166/319. Each question gets a fresh store, so this
+is not state carried between modes. The cause was not found; a candidate is
+that the GPU embeddings are not bit-for-bit identical from run to run. Hit@k
+and MRR agreed in every run that repeated a mode (hybrid 0.4/0.6 was measured
+four times: three f32 runs and one float16 run), but a last-digit change in an
+MRR, or a one-question change in recency, is within this variation.
 
 ### Full haystack — `longmemeval_s`, n=500 (the number to cite)
 
+This table is **BM25-only** (zero embeddings). With real embeddings and the
+default hybrid 0.4/0.6 the same corpus gives turn Hit@5 **81.4%** (2026-09-27;
+see [Fusion method](#fusion-method--weighted-vs-rrf-full-haystack-n500)),
+which is the headline figure.
+
 47.7 sessions and 493.5 turns per question; 4.0% of haystack sessions are evidence
 sessions, so retrieval has to actually discriminate.
 
@@ -1401,13 +1797,16 @@ the question.
 
 Real 384-d `all-MiniLM-L6-v2` embeddings, 190,015 unique texts encoded once on an
 RTX 5060 Ti (~13 min; the same work on the 8-core CPU was still unfinished after
-30 minutes, so the GPU path is not a convenience here). Turn-level:
+30 minutes, so the GPU path is not a convenience here). First measured
+2026-08-07 (c913cd1); the values below are the 2026-09-27 re-run, which moved
+the vector-only MRR and the `0.7/0.3` row (see the re-run section above).
+Turn-level:
 
 | Mode | Hit@1 | Hit@5 | Hit@10 | MRR |
 |------|-------|-------|--------|-----|
 | BM25 only (`0.0`/`1.0`) | **53.8%** | 75.0% | 81.6% | **0.6320** |
-| Vector only (`1.0`/`0.0`) | 36.0% | 71.8% | 81.6% | 0.5027 |
+| Vector only (`1.0`/`0.0`) | 36.0% | 71.8% | 81.6% | 0.5031 |
-| Hybrid (`0.7`/`0.3`) | 44.4% | **79.2%** | **86.0%** | 0.5868 |
+| Hybrid (`0.7`/`0.3`) | 44.2% | **79.2%** | **85.8%** | 0.5856 |
 
 Session-level:
 
@@ -1415,7 +1814,7 @@ Session-level:
 |------|-------|-------|--------|-----|
 | BM25 only | 86.2% | 93.6% | 96.6% | 0.8948 |
 | Vector only | 85.4% | 94.2% | 96.6% | 0.8901 |
-| Hybrid | **88.2%** | **95.8%** | **97.8%** | **0.9158** |
+| Hybrid | **88.0%** | **95.8%** | **97.6%** | **0.9146** |
 
 ### Fusion method — weighted vs. RRF, full haystack, n=500
 
@@ -1429,7 +1828,7 @@ takes a `Fusion`, and both run over the same HNSW + BM25 candidates:
 | BM25 only | **53.8%** | 75.0% | 81.6% | 0.6320 | 86.2% | 0.8948 |
 | Vector only | 36.0% | 71.8% | 81.6% | 0.5031 | 85.4% | 0.8901 |
 | **Weighted 0.4 / 0.6** | 51.6% | **81.4%** | **87.8%** | **0.6430** | **91.0%** | **0.9347** |
-| RRF (k=60) | 45.0% | 78.8% | 87.6% | 0.5967 | 89.6% | 0.9253 |
+| RRF (k=60) | 45.0% | 78.8% | 87.6% | 0.5969 | 89.6% | 0.9253 |
 
 **RRF loses to the tuned weighted sum** — 6.6pp of turn Hit@1 and 0.046 of MRR
 — and lands almost exactly where the old `0.7/0.3` weighting did (44.2% / 
@@ -1479,7 +1878,9 @@ over rank-1 precision.
 activation. Until now its combined score contained **no relevance term at
 all** — `RerankInput` did not carry the retrieval score — so a caller that
 re-ranked its candidates threw the retriever's ordering away and returned them
-ordered by age. The OpenClaw backend did exactly that on every search.
+ordered by age. `ClawhdfBackend` (the `openclaw` module) did exactly that on
+every search. (OpenClaw itself never integrated clawhdf5; see
+`docs/openclaw.md`.)
 
 Measuring that is unambiguous. "Recency" below is the share of
 `knowledge-update` questions where the newest gold session outranked the stale
@@ -1494,6 +1895,12 @@ one (see `newest_gold_first`); ~45% is chance.
 | + re-rank, relevance-led, half-life 30 days | 51.8% | 81.0% | 87.8% | 0.6427 | 51.4% |
 | + re-rank, relevance-led, half-life 90 days | **52.0%** | 80.4% | 87.8% | 0.6425 | 50.8% |
 
+Re-run on 2026-09-27 (`--rerank-sweep`, tank, commit 7a8fae0): every Hit@k and
+MRR above reproduced exactly. The recency column came out 45.3%, 87.5%,
+52.0%, 52.2%, 51.7% and 50.5%. Each of those is within one question of the
+value in the table, which is the run-to-run variation described under
+"Re-run with real embeddings" above, so the table was left as it was.
+
 **The pre-fix row is the finding.** Ordering candidates by recency alone costs
 40.6pp of Hit@1 and two thirds of MRR: the results are the newest memories in
 the pool rather than the ones that answer the question. It does ace the recency
@@ -1507,9 +1914,9 @@ cannot reach the 87.5% the degenerate ordering gets. Those two rows are the
 ends of a trade-off, and the default sits deliberately near the relevance end.
 
 **Half-life is not a sensitive knob.** Across 1, 7, 30 and 90 days recency
-moves 1.4pp and MRR 0.003 — inside the noise of a 500-question run — because
+moves 1.4pp (1.7pp in the 2026-09-27 re-run) and MRR 0.003 — inside the
-the temporal term is capped by its weight (0.3) while relevance differences
+noise of a 500-question run — because the temporal term is capped by its
-between candidates are larger. The 24-hour default is kept; there is no
+weight (0.3) while relevance differences between candidates are larger. The 24-hour default is kept; there is no
 measured reason to change it, and a corpus-matched value is not the lever it
 looks like.
 
@@ -1517,24 +1924,28 @@ looks like.
 
 `0.7/0.3` was a documented default, never a searched one. Sweeping
 `vector_weight` from 0.0 to 1.0 (`--sweep`, reusing the one-time embedding
-table) shows it is not merely suboptimal but **strictly dominated**:
+table) shows it is not merely suboptimal but **strictly dominated**. First
+measured 2026-08-07 (1537a94); the values below are the 2026-09-27 re-run,
+which changed the 0.2, 0.4, 0.5, 0.6, 0.7, 0.8 and 1.0 rows in the last digit
+or by one or two questions (see the re-run section above):
 
 | vector / keyword | Hit@1 | Hit@5 | Hit@10 | MRR | session Hit@5 |
 |---|---|---|---|---|---|
 | 0.0 / 1.0 (BM25) | **53.8%** | 75.0% | 81.6% | 0.6320 | 93.6% |
 | 0.1 / 0.9 | 53.2% | 77.4% | 83.8% | 0.6374 | 95.0% |
-| 0.2 / 0.8 | 53.6% | 78.2% | 85.6% | 0.6440 | 95.4% |
+| 0.2 / 0.8 | 53.4% | 78.0% | 85.6% | 0.6429 | 95.4% |
 | 0.3 / 0.7 | 53.2% | 78.8% | 87.2% | **0.6463** | 96.0% |
-| **0.4 / 0.6** | 51.6% | **81.4%** | 87.8% | 0.6429 | 96.8% |
+| **0.4 / 0.6** | 51.6% | **81.4%** | 87.8% | 0.6430 | 96.8% |
-| 0.5 / 0.5 | 48.2% | **81.4%** | **88.2%** | 0.6234 | **97.4%** |
+| 0.5 / 0.5 | 48.2% | **81.4%** | **88.2%** | 0.6232 | **97.4%** |
-| 0.6 / 0.4 | 46.6% | 79.8% | 87.4% | 0.6069 | 96.6% |
+| 0.6 / 0.4 | 46.6% | 79.6% | 87.4% | 0.6067 | 96.4% |
-| 0.7 / 0.3 *(old default)* | 44.4% | 79.2% | 86.0% | 0.5868 | 95.8% |
+| 0.7 / 0.3 *(old default)* | 44.2% | 79.2% | 85.8% | 0.5856 | 95.8% |
-| 0.8 / 0.2 | 40.6% | 76.2% | 85.4% | 0.5571 | 95.2% |
+| 0.8 / 0.2 | 40.6% | 76.2% | 85.4% | 0.5574 | 95.2% |
 | 0.9 / 0.1 | 37.8% | 73.4% | 84.6% | 0.5289 | 94.2% |
-| 1.0 / 0.0 (vector) | 36.0% | 71.8% | 81.6% | 0.5027 | 94.2% |
+| 1.0 / 0.0 (vector) | 36.0% | 71.8% | 81.6% | 0.5031 | 94.2% |
 
 **`0.4/0.6` beats `0.7/0.3` on every metric at both granularities** — Hit@1
-+7.2pp, Hit@5 +2.2, Hit@10 +1.8, MRR +0.056. There is no trade being made; the
++7.4pp, Hit@5 +2.2, Hit@10 +2.0, MRR +0.057 (2026-09-27 figures; +7.2pp,
++2.2, +1.8 and +0.056 as measured on 2026-08-07). There is no trade being made; the
 old default was simply on the wrong side of the peak. **`0.4/0.6` is the
 recommended setting**, with `0.3/0.7` preferable if rank-1 precision matters
 most (it takes the best MRR in the sweep and gives up only 0.6pp of Hit@1
@@ -1567,7 +1978,7 @@ worth stating plainly rather than hiding: LongMemEval questions share substantia
 vocabulary with their evidence turns, which is close to the best case for lexical
 matching, and MiniLM at 384 dimensions is a small embedding model.
 
-> **Run:** `cargo run --release --bin longmemeval_bench --features embeddings -- \
+> **Run:** `cargo run --release -p clawhdf5-bench --bin longmemeval_bench --features embeddings -- \
 >   benchmarks/longmemeval/longmemeval_s_cleaned.json --embeddings weights/all-minilm-l6-v2`
 > For the GPU path use `--features embeddings-cuda`. That requires `nvcc` on
 > `PATH` at *build* time — cudarc's build script shells out to it. The toolkit
@@ -1595,11 +2006,27 @@ price of the harder corpus, and is the reason oracle-only numbers should not be
 presented as LongMemEval results. Session-level figures on this variant are
 degenerate — see below.
 
-With real embeddings the same oracle corpus gives BM25-only 84.2% / vector-only
+With real embeddings, the same oracle corpus gives these turn-level figures
-80.4% / hybrid **85.2%** Hit@5 turn-level — hybrid ahead at Hit@5 and Hit@10 and
+(2026-09-27, tank, commit 7a8fae0; command and load in "Re-run with real
-behind at Hit@1, matching the full-haystack pattern above. (BM25-only reads 84.2%
+embeddings" above):
-here against 84.4% with zero embedding vectors: one question of 500 changes rank,
+
-with MRR identical at 0.6597. On the full haystack the two agree exactly.)
+| Mode | Hit@1 | Hit@5 | Hit@10 | MRR |
+|---|---:|---:|---:|---:|
+| BM25 only | **52.6%** | 84.4% | 90.4% | 0.6597 |
+| Vector only | 39.6% | 80.4% | 91.0% | 0.5605 |
+| Hybrid 0.4 / 0.6 (default) | 52.4% | **86.8%** | **92.4%** | **0.6678** |
+| Hybrid 0.7 / 0.3 (old default) | 48.4% | 85.2% | 92.2% | 0.6382 |
+
+Hybrid 0.4/0.6 leads at Hit@5, Hit@10 and MRR and is 0.2pp (one question)
+behind BM25 at Hit@1, as on the full haystack.
+
+Until 2026-09-27 this paragraph gave BM25-only 84.2%, vector-only 80.4% and
+hybrid **85.2%** Hit@5. Those were measured on 2026-08-07 (c913cd1), when the
+hybrid default was 0.7/0.3; today's 0.7/0.3 row reproduces the 85.2%. The
+0.4/0.6 default (29baabb) is what moves hybrid to 86.8%. The earlier BM25-only
+84.2% with embeddings, one question below the zero-embedding 84.4%, predates
+the index tie-break of 3ed0489; the two now agree, as they always did on the
+full haystack.
 
 ### Retracted: session-level recall and the MemX comparison
 
@@ -1970,7 +2397,8 @@ The tank row was measured 2026-09-24 on tank (AMD Ryzen 7 7800X3D), commit
 ### Reproducibility
 
 ```bash
-rustup override set nightly
+# Any stable toolchain at or above the MSRV (1.92) works; the original
+# 2026-07-01 run used a nightly, later runs stable.
 
 # Latency benchmarks (Criterion)
 cargo bench -p clawhdf5-agent
@@ -2073,7 +2501,9 @@ libhdf5 reads from a temp file including `open` + `read` + `close` overhead.
 | clawhdf5 hyperslab (f64, 10% slice) | — | 4.09 µs / **1.8 GiB/s** | 50.1 µs / **1.5 GiB/s** |
 
 libhdf5 f64 comparison excluded — clawhdf5's datatype encoding differs from libhdf5's (known
-gap), making cross-format reads unreliable for comparison.
+gap), making cross-format reads unreliable for comparison. (That gap was the float sign-bit
+bug, fixed 2026-09-23: `docs/known-issues.md`, "Every `f32` dataset we wrote was unreadable by
+h5py / libhdf5". The comparison has not been re-run since.)
 
 ### Chunked Read Throughput
 
@@ -2181,6 +2611,13 @@ global file mutex and flushes to disk on every attribute write or group creation
 
 ## vs libhdf5 Summary
 
+> Measured on the original i7-12650H (clawhdf5 2026-07-01, libhdf5
+> 2026-06-30). The newest run of this table is
+> [Independent Validation: tank](#independent-validation-tank-ryzen-7-7800x3d-2026-08-03)
+> (2026-08-03), which reproduces every row within ~15% except chunked
+> write (45.3x on tank; 35x on 2026-09-23 with the pure-Rust deflate, see
+> [Deflate backend](#deflate-backend-zlib-rs-vs-zlib-ng)).
+
 | Workload | clawhdf5 | libhdf5 | Speedup |
 |----------|----------|---------|---------|
 | Sequential read, 1K f32 | 634 ns | 45.2 µs | **71×** |
@@ -2212,7 +2649,7 @@ to the page cache. There is no algorithmic headroom above ~1.7 GiB/s on this har
 
 ### Caveats
 
-- libhdf5 f64 read comparison excluded — clawhdf5's f32 datatype encoding differs from libhdf5's (known compatibility gap). f64 results are clawhdf5-only.
+- libhdf5 f64 read comparison excluded — clawhdf5's f32 datatype encoding differs from libhdf5's (known compatibility gap at the time; fixed 2026-09-23, see [Sequential Read Throughput](#sequential-read-throughput)). f64 results are clawhdf5-only.
 - Serial benchmarks. clawhdf5 uses Rayon for chunk compression when > 2 chunks; that parallelism is already reflected in the chunked write numbers.
 - clawhdf5 reads from `Vec<u8>` (zero-copy from mmap in production); libhdf5 reads from a temp file. This gives clawhdf5 a structural read advantage that reflects realistic API usage.
 
@@ -2409,6 +2846,10 @@ Same not-like-for-like caveat as the "Comparison to MemX" section at the top of
 file applies — MemX's figure is end-to-end, these are a single component. Ratios are
 an order-of-magnitude indication, not a benchmark result.
 
+> The Ratio column below was retracted afterwards: see
+> [Comparison to MemX](#comparison-to-memx-arxiv260316171). Kept as recorded
+> on 2026-08-05; do not cite it.
+
 | Metric | MemX (claimed, end-to-end) | ClawhDF5 (tank, component only) | Ratio |
 |--------|----------------------------|----------------------------------|-------|
 | 100K flat search | <90 ms | 6.60 ms | ~14x |

CLAUDE.md

@@ -1,218 +1,267 @@
 # clawhdf5
 
 ## Purpose
-Pure-Rust HDF5 format implementation with HNSW vector search, WAL-backed persistence, agent memory storage, and GPU-accelerated vector search. A standalone library. Its one verified consumer is ClawBrainHub (`.brain` files); no agent framework integrates it (OpenClaw and ZeroClaw claims were withdrawn on 2026-09-25 — neither was ever true).
+Pure-Rust HDF5 implementation (read, write, in-place edit, remote and browser
+reads) plus agent memory on top of it: HNSW vector search, a WAL-backed store,
+and GPU vector distances. A standalone library. Its one verified consumer is
+ClawBrainHub (`.brain` files); no agent framework integrates it (see
+*Standing rules*).
 
 ## Architecture
 
-Cargo workspace with 18 crates under `crates/` (plus `libaec-sys`, an internal FFI bindings crate for the optional `szip` feature):
+Cargo workspace, 19 crates under `crates/` (plus `libaec-sys`, the FFI crate
+behind the optional `szip` feature). MSRV 1.92 (`rust-version`, checked in CI).
 
 | Crate | Role |
 |-------|------|
-| `clawhdf5-format` | HDF5 binary spec parser (superblock, B-tree, heap) — also holds shared type definitions and physical constants |
+| `clawhdf5-format` | The HDF5 format: parsers and writer (superblock, headers, B-trees, heaps, chunk indexes), the `Storage` trait, the filter pipeline and registry (`filter_registry`), every codec except deflate (LZ4, Zstd, SZIP, N-Bit, scale-offset, pcodec; pure-Rust LZF, bitshuffle, bzip2, Blosc 1; Blosc2 and ZFP read-only), `float16`, `checksum` |
-| `clawhdf5-io` | Read/write implementation |
+| `clawhdf5-filters` | Deflate backends (zlib-rs default, zlib-ng, Apple Compression) |
-| `clawhdf5-filters` | Deflate backends (zlib-rs, zlib-ng, Apple Compression); the HDF5 filter pipeline, the filter registry (`clawhdf5_format::filter_registry`) and the other codecs (LZ4, Zstd, SZIP, N-Bit, scale-offset, pcodec, and the pure-Rust plugin filters LZF, bitshuffle, bzip2, Blosc 1) live in `clawhdf5-format`. No Blosc2 or ZFP. |
+| `clawhdf5-io` | I/O adapters (buffers, mmap, prefetch) |
-| `clawhdf5-derive` | Proc-macro derive for HDF5-serializable structs |
+| `clawhdf5-derive` | `#[derive(H5Type)]` for compound types |
-| `clawhdf5` | Main facade crate |
+| `clawhdf5` | Facade: `File`, `FileBuilder`, `Dataset`, `FileEditor` (`src/edit/`), SWMR reading (`src/swmr.rs`) |
-| `clawhdf5-netcdf4` | NetCDF-4 compatibility layer |
+| `clawhdf5-netcdf4` | NetCDF-4 read support |
-| `clawhdf5-ann` | HNSW approximate nearest-neighbor vector index |
+| `clawhdf5-remote` | `open_url`: HTTP(S) range requests and object stores (S3, GCS, Azure) through `BlockCache` |
-| `clawhdf5-agent` | Agent memory, session history, knowledge graph storage |
+| `clawhdf5-tools` | `h5rs`: `ls`, `dump` (DDL / hdf5-json), `stat`, `diff`, `check` |
-| `clawhdf5-gpu` | GPU vector distance computation via wgpu (hand-written WGSL compute shaders) — not dataset I/O |
+| `clawhdf5-py` | PyO3 bindings (h5py-like API, remote files, `'r+'` editing) |
-| `clawhdf5-accel` | CPU SIMD acceleration path |
+| `clawhdf5-wasm` | wasm-bindgen browser reader (`open(bytes)`, `openUrl(url)`); demo in `examples/wasm-viewer/` |
-| `clawhdf5-migrate` | SQLite → HDF5 agent-memory migration |
+| `clawhdf5-ann` | HNSW index |
+| `clawhdf5-agent` | Agent memory store (`HDF5Memory`), sessions, knowledge graph, BM25 |
+| `clawhdf5-accel` | CPU SIMD kernels (AVX2, NEON) |
+| `clawhdf5-gpu` | wgpu vector distances (WGSL) — not dataset I/O; HDF5 I/O is CPU-only |
+| `clawhdf5-migrate` | SQLite → agent store migration |
+| `clawhdf5-cli` | Agent-memory CLI |
+| `clawhdf5-napi` | Node.js addon (the `packages/clawhdf5-node` wrapper is broken; `docs/known-issues.md`) |
 | `clawhdf5-android` | Android JNI bindings |
-| `clawhdf5-cli` | Command-line interface (agent memory) |
+| `clawhdf5-bench` | Benchmarks and harnesses (`search_harness`, `read_harness`, `concurrent_read`, `longmemeval_bench`, …) |
-| `clawhdf5-tools` | `h5rs`: pure-Rust HDF5 tools — `ls`, `dump` (DDL / hdf5-json), `stat`, `diff`, `check` (structural + checksum validator) |
-| `clawhdf5-napi` | Node.js native addon bindings |
-| `clawhdf5-py` | PyO3 Python bindings |
-| `clawhdf5-wasm` | WebAssembly (wasm-bindgen) reader for the browser; demo in `examples/wasm-viewer/` |
-| `clawhdf5-bench` | Benchmark suite |
 
-## Key Features
+Reference docs: `docs/known-issues.md` (open issues table first — check it
-- Zero-C-dependency HDF5 read/write: no libhdf5, and deflate defaults to
+before calling something a bug or a feature), `BENCHMARKS.md` (headline
-  pure-Rust zlib-rs (`fast-deflate` opts into zlib-ng, which needs cmake).
+numbers first), `CONFORMANCE.md` (generated), `docs/design/range-reads.md`
-  `ci-test.sh` fails if a C-building crate enters the core crates' default
+and `docs/design/swmr.md`, `CHANGELOG.md` (full detail of every fix).
-  tree. flate2 must keep `runtime_detection` with zlib-rs — without it zlib-rs
+
-  loses SIMD and inflates 3.5x slower. MSRV is 1.92 (`rust-version`, checked
+## Standing rules
-  in CI).
+
-- HNSW vector index for semantic similarity search over agent memories — the
+- **No C in the default build.** No libhdf5; deflate defaults to pure-Rust
-  `clawhdf5-agent` `hnsw` feature is **on by default**, so `hybrid_search` uses
+  zlib-rs (`fast-deflate` opts into zlib-ng, which needs cmake). `ci-test.sh`
-  the approximate `clawhdf5-ann` index for the vector stage (the index mirrors
+  fails if a C-building crate enters the core crates' default tree. Zstd,
-  the cache and self-heals on drift). Build the agent with
+  SZIP, `https` (ring) and `s3`/`gcs`/`azure` (aws-lc-rs) are opt-in. flate2
-  `--no-default-features --features float16` to force the exact linear cosine scan.
+  must keep `runtime_detection` with zlib-rs — without it zlib-rs loses SIMD
-  The agent's `parallel` feature (also default) builds the index on a thread
+  and inflates 3.5x slower.
-  pool; the graph is identical with or without it.
+- **Every file we write must open in h5py/libhdf5.** Interop tests compare
-  The index uses the HNSW paper's diversity heuristic for neighbour selection
+  against h5py and h5dump; `f32` and empty datasets did not open until
-  (plain closest-M capped recall on clustered data: 0.31 recall@10 at 100K). Its
+  2026-09-23.
-  graph is saved to `<store>.h5.ann` at each checkpoint and reloaded by `open()`
+- **float16 has one implementation:** `clawhdf5_format::float16`.
-  (tied to the checkpoint by a generation id; stale/damaged sidecars are
+- **Claims need evidence.** Performance and integration claims in docs must
-  ignored and the index rebuilt). `MemoryConfig::quantized_index` (**on by
+  be measured, dated (with machine and command), or withdrawn. Benchmark
-  default** for new stores, persisted; stores predating the setting load as
+  numbers are dated records: never edit a measured value, add a new dated
-  `false` and keep their f32 index — guarded by
+  section and mark the old one superseded.
-  `tests/fixtures/store_v2_5_0.h5`; CLI opt-out is `create --f32-index`)
+- **OpenClaw is not supported** (decided 2026-09-25): clawhdf5 is not and
-  stores the index's own copy of the embeddings as `i8`,
+  never was an OpenClaw memory plugin; the old `memory.backend = "clawhdf5"`
-  which roughly halves a loaded store's memory (2.72x -> 1.74x the raw vectors
+  config was never valid. `docs/openclaw.md` records what a real plugin would
-  at 100K); because quantised distances are approximate and `ef` cannot
+  need. The `openclaw` module's `ClawhdfBackend` is just `search` with
-  compensate, the query path then re-scores the candidate pool against the
+  re-rank + confidence on.
-  exact embeddings, which holds recall at the f32 index's level. It is also
+- **ZeroClaw does not use clawhdf5** (checked 2026-09-25 against upstream
-  faster at equal recall: 1.63x the QPS on x86-64 (AVX2) and 1.18x on a
+  v0.8.5 and the `osobh/zeroclaw` fork and their history): its memory
-  Raspberry Pi 5 (`clawhdf5_accel::dot_i8`, NEON `SDOT` via inline asm since
+  backends are its own; `clawhdf5-migrate`'s default SQLite layout is not
-  the intrinsic is unstable; plain NEON on pre-dotprod cores). The aarch64
+  ZeroClaw's schema. Don't reintroduce integration claims without an
-  code is `cfg`'d out on x86, so x86 CI never compiles or lints it — test it
+  integration and a test against the real consumer.
-  on real ARM (`rpivision02`, 10.0.2.3, is a Pi 5). `hybrid_search` keeps one incremental BM25
+- **known-issues.md:** one entry per bug; when fixed, record it in
-  index for the life of the store and never writes the store: Hebbian
+  `CHANGELOG.md` and move the entry to *Fixed (history)* with date, PR,
-  activation boosts are persisted by the next checkpoint (or on drop), not per
+  affected releases and what users must do — never delete it.
-  query. Measure any search-path change with
+
-  `cargo run --release -p clawhdf5-bench --bin search_harness` (baselines in
+## HDF5 library: invariants and gotchas
-  `BENCHMARKS.md`).
+
-- WAL (write-ahead log) for crash-safe persistence, with a chained CRC32
+- **Remote/range reads** (`docs/design/range-reads.md`, M0-M5 merged in PRs
-  trailer per entry (each entry's CRC folds in the previous entry's CRC) so a
+  #17-#19, M4 listing costs cut in #21): every format-crate read path goes through `Storage`
-  corrupted, reordered, duplicated, or spliced entry stops replay cleanly
+  (`read_at`/`read_ranges`/`hint`). `File::open_storage` takes any
-  instead of loading bad or tampered data. The pre-chaining per-entry-CRC
+  `Storage`; `clawhdf5_remote::open_url` wraps HTTP (`HttpStorage`, ureq) or
-  format (v2) is still fully readable; the oldest no-CRC format (v1) is only
+  `ObjectStoreStorage` in `BlockCache` (1 MiB blocks, LRU budget, in-flight
-  reachable through the one-time migration path in `HDF5Memory::open`, not
+  dedup, coalesced runs). Remote files are pinned by ETag/Last-Modified and
-  through the public `WalFile::read_entries`.
+  length (`RemoteError::FileChanged`). Zero-copy APIs and `File::as_bytes`
-  **What the WAL guarantees:** integrity, ordering, and recovery from a
+  need an in-memory file. Parse through `File::storage()` and the `*_in`
-  *process* crash at any point — including between a checkpoint and the WAL
+  functions, not `as_bytes`, in new code (the Python bindings do).
-  truncate (each checkpoint records a `WalMark` in `/meta`, and `open()` skips
+  `ObjectStoreStorage` runs reads on its own small tokio runtime, so it
-  the WAL prefix the `.h5` already contains, so entries are never applied
+  works from any thread.
-  twice). Checkpoints and snapshots are made durable as a unit (temp file
+- **SWMR** (`docs/design/swmr.md`): `File::open_swmr` reads a file a libhdf5
-  synced, renamed, directory synced). **What it does not guarantee:**
+  SWMR writer is appending to — positioned reads, no chunk cache, bounded
-  individual WAL appends are *not* fsynced (a deliberate latency trade-off), so
+  retries (100), `Dataset::refresh()`. clawhdf5 has no SWMR writer; remote
-  saves made since the last checkpoint can be lost on power failure or kernel
+  SWMR is out of scope.
-  panic. Current header version is 4 (adds the `Update` record used by
+- **Browser** (`clawhdf5-wasm`, read-only, no Zstd/SZIP): `openUrl` reads
-  `save_or_update`); v3 files are read and upgraded in place.
+  through the restartable "NeedBytes" cache (`src/lazy.rs`: a call is re-run
-- A store has a **single writer**: `HDF5Memory::create`/`open` hold an exclusive
+  after each wave of misses; no block is evicted while a call runs); the HTTP
-  advisory lock on `<store>.h5.lock` and a second opener gets
+  is JavaScript (`js/remote.js`).
-  `MemoryError::Locked`. Use `HDF5Memory::open_read_only` for a lock-free,
+- **In-place editing** (`clawhdf5::FileEditor`): overwrites values, grows and
-  never-writing point-in-time view (the CLI's `recall`/`stats`/`agents-md`/
+  shrinks chunked datasets (every chunk index) and sets attributes (compact
-  `export` do). An unreadable WAL (torn header, bad magic) is quarantined to
+  and dense) without rewriting the file, changing indexes and heaps as
-  `<store>.h5.wal.corrupt-<ts>` rather than blocking `open()`; a WAL with an
+  libhdf5 does; freed space is reused within one editor. Anything it cannot
-  unknown *newer* version still fails and is left untouched.
+  do safely is `Error::Unsupported` before any write (limits in
-- `MemoryConfig::float16` (**on by default** for new stores, persisted;
+  `docs/known-issues.md`). The algorithms follow libhdf5 `hdf5_1_14_6`
-  existing stores keep their recorded `false` — guarded by the v2.5.0
+  (github.com/HDFGroup/hdf5). Test changes with `cargo test -p
-  fixture in `tests/float16_store.rs`; CLI opt-out is `create --f32`) writes
+  clawhdf5-tools --test edit_interop --test edit_coverage_interop`.
-  `/memory/embeddings` as IEEE half precision (48% smaller file at 100K;
+- **Provenance:** `Dataset::verify_provenance()` (facade `provenance`
-  LongMemEval with real MiniLM embeddings identical to f32).
+  feature, default) re-hashes a dataset against its `_provenance_sha256`
-  `MemoryCache::half_precision` rounds each embedding as it enters the cache (push, update, WAL replay, and on load of a store still
+  attribute (`DatasetBuilder::with_provenance`). Opt-in per call; unkeyed
-  `f32` on disk), so memory and file agree bit for bit; the conversions live
+  hash — tamper-evident, not tamper-proof.
-  in `clawhdf5_format::float16` and must stay the single implementation.
+
-  Values beyond ±65504 are `MemoryError::InvalidEntry`. Interop: every file
+## Agent memory: invariants and gotchas
-  must open in h5py — `f32` datasets and empty datasets did not until
+
-  2026-09-23 (see `docs/known-issues.md`); the agent's `h5py_interop` test
+- **Search.** `HDF5Memory::search(query_emb, text, &SearchOptions)` is the
-  guards a whole store.
+  full path: optional source-channel filter (before ranking; exact scan of
-- `HDF5Memory::search(query_emb, text, &SearchOptions)` is the full search
+  the allowed records when cheaper than `pool × M` index distance
-  path: optional source-channel filter (applied before ranking; exact scan of
-  the allowed records whenever cheaper than `pool × M` index distance
   evaluations, and as the fallback when the pool comes back short), fusion,
   activation scaling, optional re-ranking and confidence rejection.
-  `hybrid_search`/`hybrid_search_with` are thin wrappers; `ClawhdfBackend`
+  `hybrid_search`/`hybrid_search_with` are thin wrappers. It keeps one
-  (the `openclaw` module) is `search` with re-rank + confidence on.
+  incremental BM25 index for the life of the store and never writes the
-- **OpenClaw is not supported** (decided 2026-09-25): clawhdf5 is not an
+  store: Hebbian activation boosts are persisted by the next checkpoint (or
-  OpenClaw memory plugin and never was — the old `memory.backend = "clawhdf5"`
+  on drop).
-  config was never valid. Don't reintroduce OpenClaw claims; `docs/openclaw.md`
+- **HNSW** (`hnsw` feature, default): the approximate `clawhdf5-ann` index
-  records what a real plugin would need.
+  mirrors the cache and self-heals on drift; build the agent with
-- **ZeroClaw does not use clawhdf5** (checked 2026-09-25 against upstream
+  `--no-default-features --features float16` for the exact linear scan.
-  v0.8.5 and the `osobh/zeroclaw` fork, and their full history): no
+  `parallel` (default) builds it on a thread pool with an identical graph.
-  `clawhdf5` feature or backend exists; ZeroClaw's memory backends are
+  Neighbour selection uses the HNSW paper's diversity heuristic (closest-M
-  sqlite/lucid/postgres/qdrant/markdown/none behind its own `Memory` trait.
+  capped recall at 0.31 recall@10 at 100K on clustered data). The graph is
-  `clawhdf5-migrate`'s default SQLite layout (`memory_chunks`, `sessions`,
+  saved to `<store>.h5.ann` at each checkpoint, tied to it by a generation
-  `entities`, `relations`) is not ZeroClaw's schema either (ZeroClaw's is a
+  id; a stale or damaged sidecar is ignored and the index rebuilt.
-  `memories` table). Don't reintroduce integration claims without an
+- **`MemoryConfig::quantized_index`** (default on for new stores, persisted;
-  integration and a test against the real consumer. Measure changes with
+  older stores load as `false` — guarded by `tests/fixtures/store_v2_5_0.h5`;
-  `search_harness --options-study`.
+  CLI `create --f32-index`): the index's copy of the embeddings is `i8`, and
-- `MemoryConfig::compression` is off by default; when on, embeddings are
+  the query path re-scores candidates against the exact embeddings. The
-  deflate-compressed, or Zstd with the agent's `zstd` feature (links libzstd).
+  aarch64 kernels (`clawhdf5_accel::dot_i8`, NEON `SDOT` via inline asm) are
-- Signed checkpoints (`clawhdf5-agent` `signing` module): with
+  `cfg`'d out on x86, so x86 CI never compiles them — test on real ARM
-  `HDF5Memory::set_signing_key` every checkpoint stores an Ed25519-signed
+  (`rpivision02`, 10.0.2.3, a Pi 5) or rely on the `test-arm64` job.
-  manifest (SHA-256 per record in a Merkle tree + settings/sessions/graph
+- **`MemoryConfig::float16`** (default on for new stores, persisted; older
-  hashes; per-record hashes in `/integrity/record_hashes`);
+  stores keep `false` — guarded in `tests/float16_store.rs`; CLI `create
-  `HDF5Memory::verify(path, &pk)` locates edits. The hashes must cover exactly
+  --f32`): `/memory/embeddings` is IEEE half. `MemoryCache::half_precision`
-  what the file persists in the form the loader returns it (strings lose
+  rounds each embedding as it enters the cache (push, update, WAL replay, and
-  trailing NULs; an empty WAL mark is not written) or untouched stores stop
+  load of a store still `f32` on disk) so memory and file agree bit for bit.
-  verifying — `tests/signed_store.rs` round-trips awkward strings. The key is
+  Values beyond ±65504 are `MemoryError::InvalidEntry`. The agent's
-  never persisted; a signed store refuses to checkpoint without it
+  `h5py_interop` test guards that a whole store opens in h5py.
-  (`MemoryError::SigningKeyRequired`, and `MemoryError` is `#[non_exhaustive]`).
+- **WAL.** Chained CRC32 per entry (a corrupted, reordered, duplicated or
+  spliced entry stops replay cleanly). Header version 4 (`Update` record for
+  `save_or_update`); v3 is upgraded in place, v2 read, v1 only through the
+  one-time migration in `HDF5Memory::open`. Each checkpoint records a
+  `WalMark` in `/meta` so `open()` never applies an entry twice; checkpoints
+  and snapshots are durable as a unit (temp file synced, renamed, directory
+  synced). Individual WAL appends are **not** fsynced (deliberate): saves
+  since the last checkpoint can be lost on power failure or kernel panic.
+- **Single writer.** `create`/`open` hold an exclusive lock on
+  `<store>.h5.lock` (`MemoryError::Locked` for a second opener);
+  `open_read_only` is a lock-free point-in-time view (CLI `recall`/`stats`/
+  `agents-md`/`export`). An unreadable WAL is quarantined to
+  `<store>.h5.wal.corrupt-<ts>`; a WAL of an unknown newer version fails and
+  is left untouched.
+- **Signed checkpoints** (`signing` module): with `set_signing_key` each
+  checkpoint stores an Ed25519-signed manifest (per-record SHA-256 in a
+  Merkle tree plus settings/sessions/graph hashes; `/integrity/record_hashes`);
+  `HDF5Memory::verify(path, &pk)` locates edits. The hashes must cover
+  exactly what the file persists in the form the loader returns it (strings
+  lose trailing NULs; an empty WAL mark is not written) —
+  `tests/signed_store.rs` round-trips awkward strings. The key is never
+  persisted; a signed store refuses to checkpoint without it
+  (`MemoryError::SigningKeyRequired`; `MemoryError` is `#[non_exhaustive]`).
   WAL entries after the checkpoint are not covered.
-- `Dataset::verify_provenance()` (clawhdf5 facade, `provenance` feature, on by
+- **Write bookkeeping.** `save`/`save_batch`/`save_or_update` feed an
-  default) recomputes a dataset's SHA-256 and compares it against the
+  in-memory, session-scoped provenance ledger and anomaly detector
-  `_provenance_sha256` attribute written automatically on save when
+  (`provenance.rs`, `anomaly.rs`); alerts never block a save
-  `DatasetBuilder::with_provenance` is used. It's opt-in per call, not run
+  (`take_anomaly_alerts`). `MemorySource` is inferred from the caller's
-  automatically on open — it decodes and hashes the whole dataset. The hash
+  `source_channel` string — a heuristic, not a trust boundary.
-  is unkeyed (tamper-*evident*, not tamper-*proof*): it detects accidental
+- `MemoryConfig::compression` is off by default (deflate, or Zstd with the
-  corruption, not a deliberate actor able to modify both the data and the
+  agent's `zstd` feature, which links libzstd).
-  stored hash.
-- `clawhdf5-agent`'s `HDF5Memory::save`/`save_batch`/`save_or_update` run every
-  write through an in-memory (session-scoped, not persisted to disk)
-  provenance ledger and write-anomaly detector: a content hash per record
-  (`provenance.rs`) for detecting accidental mid-session corruption, plus
-  rate-limit/injection-pattern/source-distribution checks (`anomaly.rs`).
-  Alerts never block a save — drain them with `HDF5Memory::take_anomaly_alerts`.
-  `MemorySource` for this bookkeeping is inferred from the caller-supplied
-  `source_channel` string (a heuristic, not an authenticated trust boundary).
-- GPU-accelerated vector distance computation (`clawhdf5-gpu`, wgpu); HDF5 I/O itself is CPU-only
-- Browser: `clawhdf5-wasm` (wasm-bindgen, read-only, file held in memory;
-  no Zstd/SZIP since they link C) and the `examples/wasm-viewer/` page.
-  `examples/wasm-viewer/test/run.sh` builds the package (needs the
-  `wasm-bindgen` CLI at the crate's exact version) and tests it under Node
-  and headless Chromium (a Playwright download in `~/.cache/ms-playwright`
-  on tank); the CI container has neither, so CI runs the native
-  `clawhdf5-wasm` `h5py_interop` test on the same fixture. Size numbers are
-  in the example's README.
-- Python and Node.js bindings for cross-language use
-- NetCDF-4 compatibility for scientific data interop
 
 ## Workflows
 
-### Build
+Put `$HOME/.cargo/bin` on `PATH`. The h5py/netCDF4 interop tests find their
+Python through `CLAWHDF5_PYTHON` (or `.venv/bin/python`); create it with
+`python3 -m venv .venv && .venv/bin/pip install h5py numpy netCDF4 hdf5plugin`.
+Set `CLAWHDF5_REQUIRE_INTEROP=1` to make a missing interpreter a failure.
+
 ```bash
 cargo build --release
-```
-
-### Test
-```bash
 cargo test --workspace
+bash scripts/ci-test.sh          # everything CI runs (see below)
 ```
 
-### CI
+### CI (`.gitea/workflows/`)
-`.gitea/workflows/ci.yml` has two jobs, both green as of 2026-09-22:
+- **`ci.yml` `test`** (`ubuntu-latest`, `rust:latest` container; runners
-- **`test`** (`ubuntu-latest`, in `rust:latest`) runs `scripts/ci-test.sh` with
+  `tank`, `architect`): installs h5py/netCDF4/xarray/hdf5plugin/maturin/pytest,
-  the h5py/netCDF4 interop suites required (`CLAWHDF5_REQUIRE_INTEROP=1`).
+  `hdf5-tools` and `cmake`, then runs `scripts/ci-test.sh` with
-  Served by the `tank` and `architect` runners.
+  `CLAWHDF5_REQUIRE_INTEROP=1`. The script runs: fmt; clippy (workspace, the
-- **`test-arm64`** (`linux_arm64`) lints and tests the aarch64 code — the NEON
+  format feature matrix, each plugin filter alone, parallel, fast-deflate,
-  kernels are `cfg`'d out on x86, so this is the only place they are built.
+  remote with all backends, h5rs remote); "no C in the default build";
-  Served by `vision-01` (host mode) and `vision-02` (Docker), so steps must
+  wasm32 build and clippy; `check-32bit-casts.sh`; the wasm package under
-  work in both.
+  Node when `node` and `wasm-bindgen` exist (not in CI); the MSRV check;
+  `cargo test` (workspace plus feature variants: format matrix, parallel,
+  remote/object_store, h5rs URLs, ann parallel, fast-deflate); the h5py
+  interop suites (`writer_h5py_tests --include-ignored`, plugin filters,
+  ZFP); the Python package (clippy, `maturin build`, pytest vs h5py);
+  `cargo bench --no-run`; `check-nostd.sh`; an optional fuzz smoke run
+  (`CLAWHDF5_FUZZ_SECONDS`).
+- **`ci.yml` `test-arm64`** (`linux_arm64`; `vision-01` host mode,
+  `vision-02` Docker — steps must work in both): clippy of
+  `clawhdf5-accel`, tests of `-accel`, `-ann`, `-format`; the only place the NEON kernels build.
+- **`conformance.yml`** (nightly 03:17 UTC and manual): probe unit tests,
+  `conformance/test_ref.py`, then `conformance/run.sh` (gate:
+  `conformance/check.py` against `baseline.json`).
 
-Keep workflows free of JavaScript actions (`actions/checkout`, `actions/cache`,
+Keep workflows free of JavaScript actions (`actions/checkout`,
-…): `rust:latest` has no `node`, and not every runner reaches GitHub, where
+`actions/cache`, …): `rust:latest` has no `node` and not every runner reaches
-they are fetched from. Check out with plain `git` instead. The `test` job
+GitHub. Check out with plain `git`. Runners are `gitea-runner` 3.5.0 from
-installs `cmake` for the opt-in `fast-deflate` (zlib-ng) steps; the default
+`docker.gitea.com/act_runner` (`gitea/act_runner:latest` on Docker Hub is
-build needs no C toolchain, so `test-arm64` does not.
+frozen at 0.6.1).
-All runners are on `gitea-runner` 3.5.0, from `docker.gitea.com/act_runner`
-— `gitea/act_runner:latest` on Docker Hub is frozen at 0.6.1.
 
-### CLI
+### Conformance
 ```bash
-cargo run -p clawhdf5-cli -- --help
+CLAWHDF5_PYTHON=.venv/bin/python bash conformance/run.sh --no-fetch   # writes CONFORMANCE.md
-# create, save, search, recall, stats, flush-wal, agents-md, export, snapshot subcommands
 ```
+Reads 697 files of eight pinned corpora with clawhdf5 and h5py and compares
+them object by object (602 ok in the run of 2026-09-28). `CONFORMANCE.md` is
+generated — never hand-edit it (its wording lives in `conformance/report.py`).
+Use `--update-baseline` only after an intended change in results.
+`CONFORMANCE_CACHE` points at an existing corpus cache (`conformance/.cache`,
+about 450 MB). See `conformance/README.md`.
 
-### HDF5 tools (`h5rs`, crate `clawhdf5-tools`)
+### HDF5 tools (`h5rs`)
 ```bash
 cargo run -p clawhdf5-tools -- ls -r file.h5    # also dump [--json], stat, diff, check
 bash scripts/h5rs-fuzz.sh                       # every subcommand over the CVE corpus: no panic/crash/hang
 bash scripts/h5rs-check-ok-files.sh --data      # check passes every fully-read conformance file
 ```
-Its interop tests compare against h5ls/h5stat/h5dump/h5diff (Debian
+Interop tests compare against h5ls/h5stat/h5dump/h5diff (Debian `hdf5-tools`);
-`hdf5-tools`, installed in CI); `dump` must stay byte-identical to h5dump on
+`dump` must stay byte-identical to h5dump on the test files.
-the test files.
+
+### Remote and browser tests
+- `clawhdf5-remote` tests run a std-only HTTP server
+  (`tests/common/server.rs`, also the `range_server` example);
+  `CLAWHDF5_REMOTE_CORPUS=conformance/.cache/corpus` compares every corpus
+  file over HTTP with `File::open`.
+- wasm: `bash examples/wasm-viewer/test/run.sh` builds the package (needs the
+  `wasm-bindgen` CLI at the crate's exact version) and tests it under Node and
+  headless Chromium (Playwright's download in `~/.cache/ms-playwright` on
+  tank) against `test/serve.py` (range server with request counts). CI has
+  neither, so it runs the native `h5py_interop` and `lazy` tests
+  (`CLAWHDF5_WASM_CORPUS=conformance/.cache/corpus` for the corpus).
 
 ### Python bindings
 ```bash
-cd crates/clawhdf5-py
+cd crates/clawhdf5-py && maturin develop
-maturin develop
+python -m pytest crates/clawhdf5-py/tests      # compares with h5py; editing tests want CLAWHDF5_H5RS=<path to h5rs>
-python -c "import clawhdf5; print(clawhdf5.__version__)"
+```
+
+### Benchmarks
+- Search path: `cargo run --release -p clawhdf5-bench --bin search_harness`
+  (`--full`, `--options-study`, `--footprint`, …); reads: `read_harness`,
+  `concurrent_read`; criterion benches with `cargo bench -p <crate>`.
+- Run on an idle machine (1-minute load average below 2; wait otherwise),
+  alternate base and candidate binaries for A/B comparisons, and record date,
+  machine, commit and command with every number in `BENCHMARKS.md`.
+- `BENCHMARKS.md` is written by hand from dated runs; no script regenerates
+  it (the old `scripts/run-benchmarks.sh`, which benchmarked the pre-rename
+  `rustyhdf5-format` and overwrote the file, was removed on 2026-09-28).
+
+### CLI
+```bash
+cargo run -p clawhdf5-cli -- --help
+# create, save, search, recall, stats, flush-wal, agents-md, export, snapshot, keygen, verify
 ```
 
 ## Integration
@@ -220,8 +269,7 @@ python -c "import clawhdf5; print(clawhdf5.__version__)"
   verified consumer: `cbh-core` reads and writes `.brain` files through the
   facade (`File`, `FileBuilder`, `AttrValue`, `Selection`), `cbh-scanner`
   uses the facade, and `cbh-cli` uses `clawhdf5_agent::bm25::BM25Index`. It
-  depends on this repo by path (`../clawhdf5`), so it builds against whatever
+  depends on this repo by path (`../clawhdf5`), so changes to those APIs
-  is checked out — changes to those APIs reach it directly. Verified
+  reach it directly. Verified 2026-09-25 against main: builds, and its 204
-  2026-09-25 against main: builds, and its 204 tests pass.
+  tests pass.
-- OpenClaw and ZeroClaw were both described as consumers; neither integrates
+- OpenClaw and ZeroClaw integrate nothing (see *Standing rules*).
-  clawhdf5 (see Key Features and `docs/openclaw.md`).

CONFORMANCE.md

@@ -13,15 +13,15 @@ fatal. This file is generated by `conformance/run.sh`; do not edit it by hand.
 
 | | |
 |---|---|
-| date | 2026-09-26 14:18 UTC |
+| date | 2026-09-28 04:29 UTC |
-| clawhdf5 commit | `73a01f1256fb9bf1b1e7601f755af9e8273cec4e` |
+| clawhdf5 commit | `bf5a163dcf7fe28d651ada6545d8136ffeffc825` |
 | machine | `tank`: AMD Ryzen 7 7800X3D 8-Core Processor, 16 CPUs, 61 GiB, Linux 7.0.0-34-generic x86_64 |
 | command | `conformance/run.sh --no-fetch --update-baseline` |
 | rustc | rustc 1.98.1 (48a229cea 2026-09-01) |
 | reference | h5py 3.16.0, HDF5 2.0.0, numpy 2.5.3, hdf5plugin 7.1.0, Python 3.14.4 |
 | h5dump | Version 1.14.6 (CVE corpus only) |
 | limits | 20 s timeout (SIGKILL), 4096 MiB address space, per process; 16 files in parallel |
-| runtime | 23 s probing + comparing (0 s fetch/build before it) |
+| runtime | 20 s probing + comparing (5 s fetch/build before it) |
 
 ## Results
 
@@ -29,23 +29,24 @@ A file's class is the first that applies:
 
 - **panic / hang / crash / oom** — clawhdf5 panicked (caught per object or not), hit the timeout, died on a signal, or failed an allocation. The CI gate fails on any of these.
 - **h5py-cannot-read** — libhdf5 could not open the file (or itself crashed or hung). Nothing to compare against; most are the deliberately malformed CVE reproducers.
+- **ref-bug** — every difference is an object clawhdf5 refuses that h5py reads only through a libhdf5 bug: the values h5py returns for it change with the reading process's heap, re-checked in every run (see *Reference bugs*).
 - **our-error** — clawhdf5 returned an error for something h5py reads.
 - **mismatch** — both read it, but the shapes, values, object set or attribute set differ.
 - **ok** — every object h5py reads, clawhdf5 reads identically.
 
-| corpus | files | ok | our-error | mismatch | h5py-cannot-read | panic | hang | crash | oom |
+| corpus | files | ok | our-error | mismatch | h5py-cannot-read | ref-bug | panic | hang | crash | oom |
-|---|---|---|---|---|---|---|---|---|---|
+|---|---|---|---|---|---|---|---|---|---|---|
-| NCAS-CMS_pyfive | 33 | 32 | 0 | 1 | 0 | 0 | 0 | 0 | 0 |
+| NCAS-CMS_pyfive | 33 | 33 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 |
-| cve_hdf5 | 147 | 100 | 6 | 9 | 32 | 0 | 0 | 0 | 0 |
+| cve_hdf5 | 147 | 113 | 0 | 0 | 32 | 2 | 0 | 0 | 0 | 0 |
-| h5py_data | 4 | 4 | 0 | 0 | 0 | 0 | 0 | 0 | 0 |
+| h5py_data | 4 | 4 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 |
-| hdf5 | 466 | 392 | 4 | 10 | 60 | 0 | 0 | 0 | 0 |
+| hdf5 | 466 | 405 | 1 | 0 | 60 | 0 | 0 | 0 | 0 | 0 |
-| netcdf-c | 20 | 20 | 0 | 0 | 0 | 0 | 0 | 0 | 0 |
+| netcdf-c | 20 | 20 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 |
-| netcdf4-python | 18 | 18 | 0 | 0 | 0 | 0 | 0 | 0 | 0 |
+| netcdf4-python | 18 | 18 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 |
-| usnistgov_h5wasm | 5 | 5 | 0 | 0 | 0 | 0 | 0 | 0 | 0 |
+| usnistgov_h5wasm | 5 | 5 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 |
-| xarray-data | 4 | 4 | 0 | 0 | 0 | 0 | 0 | 0 | 0 |
+| xarray-data | 4 | 4 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 |
-| **all** | **697** | **575** | **10** | **20** | **92** | **0** | **0** | **0** | **0** |
+| **all** | **697** | **602** | **1** | **0** | **92** | **2** | **0** | **0** | **0** | **0** |
 
-2 of the 20 mismatches are a known h5py bug, not ours (see *Known not-our-bug*).
+**Our errors and mismatches: 1.** Files not ok: 1 our-error, 92 h5py-cannot-read, 2 ref-bug. 2 object(s) were compared against h5py's values corrected for a known h5py bug (2 identical to clawhdf5's; see *Reference bugs*).
 
 Corpora (fetched by `conformance/fetch-corpus.sh` into the gitignored `conformance/.cache/`):
 
@@ -70,27 +71,11 @@ Grouped by normalised error message. *files* counts files whose class this cause
 
 | files | objects | error | examples |
 |---:|---:|---|---|
-| 3 | 3 | `DataSizeMismatch { expected: N, actual: N }` | `cve_hdf5/cvefiles/cve-2020-18494.h5`, `cve_hdf5/cvefiles/cve-2024-32623.h5`, `cve_hdf5/cvefiles/cve-2025-2309.h5` |
+| 1 | 1 | `ChunkedReadError("…")` | `hdf5/test/testfiles/bad_nbit_parms_walk.h5` |
-| 2 | 2 | `ChunkedReadError("…")` | `cve_hdf5/cvefiles/cve-2025-2308.h5`, `hdf5/test/testfiles/bad_nbit_parms_walk.h5` |
-| 2 | 2 | `UnsupportedFilter(N)` | `hdf5/HDF5Examples/C/H5FLT/tfiles/h5ex_d_blosc2.h5`, `hdf5/HDF5Examples/C/H5FLT/tfiles/h5ex_d_zfp.h5` |
-| 1 | 1 | `UnexpectedEof { expected: N, available: N }` | `cve_hdf5/cvefiles/cve-2019-9151.h5` |
-| 1 | 1 | `MissingMessage(Dataspace)` | `cve_hdf5/cvefiles/cve-2024-33874.h5` |
-| 1 | 1 | `InvalidObjectHeaderVersion(N)` | `hdf5/tools/test/testfiles/h5clear_mdc_image.h5` |
 
 ## Mismatch root causes
 
-| files | objects | cause | examples |
+None.
-|---:|---:|---|---|
-| 13 | 14 | `missing-object` | `cve_hdf5/cvefiles/cve-2019-8397.h5`, `cve_hdf5/cvefiles/cve-2019-8398.h5`, `cve_hdf5/cvefiles/cve-2021-46243.h5` (+10 more) |
-| 2 | 6 | `extra-attr` | `cve_hdf5/cvefiles/cve-2018-17438`, `cve_hdf5/cvefiles/cve-2018-17439` |
-| 1 | 1 | `attr-values: ours=vlen(>u8) h5py=object layout=- filters=-` | `NCAS-CMS_pyfive/tests/data/attr_datatypes.hdf5` |
-| 1 | 4 | `extra-object` | `cve_hdf5/cvefiles/cve-2021-46244.h5` |
-| 1 | 1 | `values: ours=<f4 h5py=float32 layout=chunked filters=-` | `cve_hdf5/cvefiles/cve-2025-44904.h5` |
-| 1 | 1 | `values: ours=>i2 h5py=>i2 layout=chunked filters=[6]` | `cve_hdf5/cvefiles/cve-2025-44905.h5` |
-| 1 | 1 | `values: ours=>f4 h5py=>f4 layout=chunked filters=[2]` | `cve_hdf5/cvefiles/cve-2025-44905.h5` |
-| 1 | 1 | `values: ours=<f4 h5py=float32 layout=chunked filters=[2]` | `cve_hdf5/cvefiles/cve-2025-44905.h5` |
-| 1 | 1 | `values: ours=((<i4)[6, 3])[4] h5py=(('<i4', (6, 3)), (4,)) layout=contiguous filters=-` | `hdf5/tools/test/testfiles/tarray3.h5` |
-| 1 | 1 | `values: ours=vlen({r:>f4,i:>f4}8) h5py=object layout=contiguous filters=-` | `hdf5/tools/test/testfiles/tcomplex_be.h5` |
 
 ## CVE corpus: clawhdf5 vs h5dump vs h5py
 
@@ -102,7 +87,7 @@ columns are.
 
 | tool | read | error | panic | crash | hang | oom |
 |---|---:|---:|---:|---:|---:|---:|
-| clawhdf5 | 140 | 7 | 0 | 0 | 0 | 0 |
+| clawhdf5 | 121 | 26 | 0 | 0 | 0 | 0 |
 | h5dump 1.14.6 | 16 | 129 | 0 | 2 | 0 | 0 |
 | h5py 3.16.0 / HDF5 2.0.0 | 115 | 31 | 0 | 1 | 0 | 0 |
 
@@ -156,27 +141,27 @@ columns are.
 | cvefiles/cve-2018-17435.h5 | error exit | read 3 obj, 1 errors | read 3 obj, 1 errors | ok |
 | cvefiles/cve-2018-17436 | error exit | open error | open error | h5py-cannot-read |
 | cvefiles/cve-2018-17437.h5 | error exit | read 3 obj, 1 errors | read 3 obj, 1 errors | ok |
-| cvefiles/cve-2018-17438 | error exit | read 3 obj, 1 errors | read 3 obj, 1 errors | mismatch |
+| cvefiles/cve-2018-17438 | error exit | read 3 obj, 1 errors | read 3 obj, 1 errors | ok |
-| cvefiles/cve-2018-17439 | error exit | read 3 obj, 1 errors | read 3 obj, 1 errors | mismatch |
+| cvefiles/cve-2018-17439 | error exit | read 3 obj, 1 errors | read 3 obj, 1 errors | ok |
 | cvefiles/cve-2019-8396.h5 | error exit | read 3 obj, 2 errors | read 3 obj, 2 errors | ok |
-| cvefiles/cve-2019-8397.h5 | error exit | read 3 obj, 2 errors | read 2 obj, 1 errors | mismatch |
+| cvefiles/cve-2019-8397.h5 | error exit | read 2 obj, 1 errors | read 2 obj, 1 errors | ok |
-| cvefiles/cve-2019-8398.h5 | error exit | read 3 obj, 2 errors | read 2 obj, 1 errors | mismatch |
+| cvefiles/cve-2019-8398.h5 | error exit | read 2 obj, 1 errors | read 2 obj, 1 errors | ok |
-| cvefiles/cve-2019-9151.h5 | error exit | read 3 obj, 1 errors | read 3 obj, 2 errors | our-error |
+| cvefiles/cve-2019-9151.h5 | error exit | read 3 obj, 1 errors | read 3 obj, 2 errors | ok |
 | cvefiles/cve-2019-9152.h5 | error exit | read 3 obj, 1 errors | read 3 obj, 1 errors | ok |
 | cvefiles/cve-2020-10809 | error exit | open error | open error | h5py-cannot-read |
-| cvefiles/cve-2020-10810.h5 | error exit | open error | read 2 obj | h5py-cannot-read |
+| cvefiles/cve-2020-10810.h5 | error exit | open error | open error | h5py-cannot-read |
 | cvefiles/cve-2020-10811.h5 | error exit | read 25 obj, 1 errors | read 25 obj, 1 errors | ok |
-| cvefiles/cve-2020-10812.h5 | error exit | open error | read 2 obj | h5py-cannot-read |
+| cvefiles/cve-2020-10812.h5 | error exit | open error | open error | h5py-cannot-read |
 | cvefiles/cve-2020-18232.h5 | error exit | read 3 obj, 2 errors | read 3 obj, 2 errors | ok |
-| cvefiles/cve-2020-18494.h5 | ok | read 2 obj | read 2 obj, 1 errors | our-error |
+| cvefiles/cve-2020-18494.h5 | ok | read 2 obj | read 2 obj | ok |
 | cvefiles/cve-2021-36977.h5 | error exit | read 1 obj, 1 errors | read 1 obj, 1 errors | ok |
 | cvefiles/cve-2021-37501.h5 | error exit | read 18 obj, 1 errors | read 18 obj, 1 errors | ok |
 | cvefiles/cve-2021-45829.h5 | error exit | read 1 obj, 2 errors | read 1 obj, 1 errors | ok |
-| cvefiles/cve-2021-45830.h5 | error exit | open error | read 1 obj, 1 errors | h5py-cannot-read |
+| cvefiles/cve-2021-45830.h5 | error exit | open error | open error | h5py-cannot-read |
 | cvefiles/cve-2021-45833.h5 | error exit | read 2 obj, 1 errors | read 2 obj, 1 errors | ok |
 | cvefiles/cve-2021-46242.h5 | error exit | open error | read 1 obj, 1 errors | h5py-cannot-read |
-| cvefiles/cve-2021-46243.h5 | error exit | read 3 obj, 2 errors | read 2 obj, 1 errors | mismatch |
+| cvefiles/cve-2021-46243.h5 | error exit | read 2 obj, 1 errors | read 2 obj, 1 errors | ok |
-| cvefiles/cve-2021-46244.h5 | error exit | read 2 obj, 1 errors | read 6 obj, 4 errors | mismatch |
+| cvefiles/cve-2021-46244.h5 | error exit | read 2 obj, 1 errors | read 2 obj, 1 errors | ok |
 | cvefiles/cve-2024-29157.h5 | error exit | read 4 obj, 7 errors | read 4 obj, 7 errors | ok |
 | cvefiles/cve-2024-29158.h5 | ok | read 3 obj, 1 errors | read 3 obj, 1 errors | ok |
 | cvefiles/cve-2024-29159.h5 | error exit | read 2 obj, 1 errors | read 2 obj, 1 errors | ok |
@@ -201,50 +186,50 @@ columns are.
 | cvefiles/cve-2024-32615.h5 | error exit | read 4 obj, 1 errors | read 4 obj, 1 errors | ok |
 | cvefiles/cve-2024-32616.h5 | error exit | read 10 obj, 7 errors | read 10 obj, 6 errors | ok |
 | cvefiles/cve-2024-32617.h5 | error exit | read 1 obj, 1 errors | read 1 obj, 1 errors | ok |
-| cvefiles/cve-2024-32618.h5 | error exit | read 4 obj, 2 errors | read 3 obj, 1 errors | mismatch |
+| cvefiles/cve-2024-32618.h5 | error exit | read 3 obj, 1 errors | read 3 obj, 1 errors | ok |
 | cvefiles/cve-2024-32619.h5 | error exit | read 3 obj, 2 errors | read 3 obj, 2 errors | ok |
 | cvefiles/cve-2024-32620.h5 | error exit | read 1 obj, 1 errors | read 1 obj, 1 errors | ok |
 | cvefiles/cve-2024-32621.h5 | ok | read 3 obj, 1 errors | read 3 obj, 1 errors | ok |
 | cvefiles/cve-2024-32622.h5 | ok | read 3 obj, 1 errors | read 3 obj, 1 errors | ok |
-| cvefiles/cve-2024-32623.h5 | ok | read 6 obj | read 6 obj, 1 errors | our-error |
+| cvefiles/cve-2024-32623.h5 | ok | read 6 obj | read 6 obj | ok |
-| cvefiles/cve-2024-32624.h5 | error exit | read 6 obj, 1 errors | read 6 obj | ok |
+| cvefiles/cve-2024-32624.h5 | error exit | read 6 obj, 1 errors | read 6 obj, 1 errors | ok |
 | cvefiles/cve-2024-33873.h5 | error exit | read 4 obj, 1 errors | read 4 obj, 1 errors | ok |
-| cvefiles/cve-2024-33874.h5 | ok | read 6 obj, 1 errors | read 6 obj, 2 errors | our-error |
+| cvefiles/cve-2024-33874.h5 | ok | read 6 obj, 1 errors | read 6 obj, 1 errors | ok |
 | cvefiles/cve-2024-33875.h5 | ok | read 2 obj | read 2 obj | ok |
 | cvefiles/cve-2024-33876.h5 | ok | read 3 obj, 1 errors | read 3 obj, 1 errors | ok |
 | cvefiles/cve-2024-33877.h5 | error exit | read 8 obj, 1 errors | read 8 obj, 1 errors | ok |
 | cvefiles/cve-2025-2153.h5 | error exit | open error | read 1 obj, 1 errors | h5py-cannot-read |
-| cvefiles/cve-2025-2308.h5 | error exit | read 25 obj, 1 errors | read 25 obj, 2 errors | our-error |
+| cvefiles/cve-2025-2308.h5 | error exit | read 25 obj, 1 errors | read 25 obj, 2 errors | ref-bug |
-| cvefiles/cve-2025-2309.h5 | ok | read 6 obj, 1 errors | read 6 obj, 1 errors | our-error |
+| cvefiles/cve-2025-2309.h5 | ok | read 6 obj, 1 errors | read 6 obj | ok |
 | cvefiles/cve-2025-2310.h5 | error exit | read 24 obj, 8 errors | read 24 obj, 8 errors | ok |
-| cvefiles/cve-2025-2912.h5 | error exit | open error | read 1 obj, 1 errors | h5py-cannot-read |
+| cvefiles/cve-2025-2912.h5 | error exit | open error | open error | h5py-cannot-read |
-| cvefiles/cve-2025-2913.h5 | error exit | open error | read 1 obj, 1 errors | h5py-cannot-read |
+| cvefiles/cve-2025-2913.h5 | error exit | open error | open error | h5py-cannot-read |
-| cvefiles/cve-2025-2914.h5 | error exit | open error | read 1 obj, 1 errors | h5py-cannot-read |
+| cvefiles/cve-2025-2914.h5 | error exit | open error | open error | h5py-cannot-read |
-| cvefiles/cve-2025-2915.h5 | error exit | open error | read 1 obj, 1 errors | h5py-cannot-read |
+| cvefiles/cve-2025-2915.h5 | error exit | open error | open error | h5py-cannot-read |
-| cvefiles/cve-2025-2923.h5 | error exit | open error | read 1 obj, 1 errors | h5py-cannot-read |
+| cvefiles/cve-2025-2923.h5 | error exit | open error | open error | h5py-cannot-read |
 | cvefiles/cve-2025-2924.h5 | error exit | read 1 obj, 1 errors | read 1 obj, 1 errors | ok |
 | cvefiles/cve-2025-2925.h5 | error exit | read 1 obj, 1 errors | read 1 obj, 1 errors | ok |
-| cvefiles/cve-2025-2926.h5 | error exit | open error | read 1 obj, 1 errors | h5py-cannot-read |
+| cvefiles/cve-2025-2926.h5 | error exit | open error | open error | h5py-cannot-read |
-| cvefiles/cve-2025-44904.h5 | error exit | read 25 obj, 1 errors | read 25 obj, 1 errors | mismatch |
+| cvefiles/cve-2025-44904.h5 | error exit | read 25 obj, 1 errors | read 25 obj, 2 errors | ref-bug |
-| cvefiles/cve-2025-44905.h5 | error exit | read 25 obj, 3 errors | read 25 obj, 3 errors | mismatch |
+| cvefiles/cve-2025-44905.h5 | error exit | read 25 obj, 3 errors | read 25 obj, 3 errors | ok |
 | cvefiles/cve-2025-6269-1.h5 | error exit | read 1 obj, 1 errors | read 1 obj, 1 errors | ok |
 | cvefiles/cve-2025-6269-2.h5 | error exit | read 1 obj, 1 errors | read 1 obj, 1 errors | ok |
 | cvefiles/cve-2025-6269-3.h5 | error exit | read 1 obj, 1 errors | read 1 obj, 1 errors | ok |
 | cvefiles/cve-2025-6269-4.h5 | error exit | read 1 obj, 1 errors | read 1 obj, 1 errors | ok |
-| cvefiles/cve-2025-6270-1.h5 | error exit | open error | read 1 obj, 1 errors | h5py-cannot-read |
+| cvefiles/cve-2025-6270-1.h5 | error exit | open error | open error | h5py-cannot-read |
-| cvefiles/cve-2025-6270-2.h5 | error exit | open error | read 1 obj, 1 errors | h5py-cannot-read |
+| cvefiles/cve-2025-6270-2.h5 | error exit | open error | open error | h5py-cannot-read |
-| cvefiles/cve-2025-6270-3.h5 | error exit | open error | read 1 obj, 1 errors | h5py-cannot-read |
+| cvefiles/cve-2025-6270-3.h5 | error exit | open error | open error | h5py-cannot-read |
 | cvefiles/cve-2025-6516.h5 | error exit | read 1 obj, 1 errors | read 1 obj, 1 errors | ok |
 | cvefiles/cve-2025-6750.h5 | error exit | open error | read 1 obj, 1 errors | h5py-cannot-read |
-| cvefiles/cve-2025-6816.h5 | error exit | open error | read 1 obj, 1 errors | h5py-cannot-read |
+| cvefiles/cve-2025-6816.h5 | error exit | open error | open error | h5py-cannot-read |
-| cvefiles/cve-2025-6817.h5 | error exit | open error | read 1 obj | h5py-cannot-read |
+| cvefiles/cve-2025-6817.h5 | error exit | open error | open error | h5py-cannot-read |
-| cvefiles/cve-2025-6818.h5 | error exit | open error | read 1 obj | h5py-cannot-read |
+| cvefiles/cve-2025-6818.h5 | error exit | open error | open error | h5py-cannot-read |
-| cvefiles/cve-2025-6856.h5 | error exit | open error | read 1 obj, 1 errors | h5py-cannot-read |
+| cvefiles/cve-2025-6856.h5 | error exit | open error | open error | h5py-cannot-read |
 | cvefiles/cve-2025-6857.h5 | error exit | read 1 obj, 1 errors | read 1 obj, 1 errors | ok |
-| cvefiles/cve-2025-6858.h5 | SIGSEGV | open error | read 1 obj, 1 errors | h5py-cannot-read |
+| cvefiles/cve-2025-6858.h5 | SIGSEGV | open error | open error | h5py-cannot-read |
 | cvefiles/cve-2025-7067.h5 | error exit | read 1 obj, 1 errors | read 1 obj, 1 errors | ok |
-| cvefiles/cve-2025-7068.h5 | error exit | open error | read 1 obj, 1 errors | h5py-cannot-read |
+| cvefiles/cve-2025-7068.h5 | error exit | open error | open error | h5py-cannot-read |
-| cvefiles/cve-2025-7069.h5 | error exit | open error | read 1 obj, 1 errors | h5py-cannot-read |
+| cvefiles/cve-2025-7069.h5 | error exit | open error | open error | h5py-cannot-read |
 | cvefiles/cve-2026-26200.h5 | error exit | read 1 obj, 1 errors | read 1 obj, 1 errors | ok |
 | cvefiles/cve-2026-34734.h5 | error exit | read 2 obj, 1 errors | read 2 obj, 1 errors | ok |
 | cvefiles/cve-2026-92627.h5 | error exit | read 2 obj, 1 errors | read 2 obj, 1 errors | ok |
@@ -260,13 +245,36 @@ columns are.
 
 </details>
 
-## Known not-our-bug
+## Reference bugs
+
+### Objects h5py reads only through a libhdf5 bug (*ref-bug*)
+
+clawhdf5 refuses these objects; h5py 3.16 / HDF5 2.0 returns values for them. `conformance/ref_bugs.py`
+re-reads each with h5py in six fresh processes whose heaps differ (h5py imported before numpy, three
+times and twice more with `MALLOC_PERTURB_`, and numpy imported first). Values the file determines
+come out the same every time; these do not, so they are memory libhdf5 over-reads, not the file's
+data. A file is *ref-bug* only while every one of its differences is such an object confirmed in
+the same run; an object that reads the same every time goes back to *our-error*. Reproducer:
+`python conformance/ref_bugs.py conformance/.cache/corpus` (prints every read's outcome).
+
+| file | object | distinct results in 6 reads | confirmed | what goes wrong |
+|---|---|---:|---|---|
+| `cve_hdf5/cvefiles/cve-2025-2308.h5` | `/Scale_offset_long_long_data_le` | 6 | yes | the first chunk records minbits 11: its 12 values need 17 bytes of codes, and the 26-byte chunk holds 5 after its 21-byte header; libhdf5's scale-offset decoder reads past its buffer, and develop refuses the chunk ("Buffer too short") |
+| `cve_hdf5/cvefiles/cve-2025-44904.h5` | `/Scale_offset_float_data_le` | 6 | yes | unfiltered chunks stored as 38 and 37 bytes for 48-byte chunks: 1.14/2.0 read the stored bytes into a buffer of that size and use it as the whole chunk (H5D__chunk_lock), so the rest is heap memory; develop refuses them ("incorrect chunk size returned from index for unfiltered chunk") |
+| `hdf5/test/testfiles/bad_nbit_parms_walk.h5` | `/Nbit_int_data_le` | 1 | **no** | the N-Bit parameter list holds 7 values (cd_values[0] = 7) where an integer needs 8: the decoder takes the bit offset from cd_values[7], past the list; libhdf5's own test (`test_filter_bad_params`, test/dsets.c on develop) requires the read to fail |
+
+### Values corrected for a known h5py bug
 
 - **h5py big-endian variable-length sequences.** h5py returns the elements of a VL sequence
   whose base type is big-endian with the file's big-endian bytes but a native (little-endian)
-  numpy dtype, so the values it reports are byte-swapped garbage; `h5dump` prints the values
+  numpy dtype: a `h5py.vlen_dtype(np.dtype('>f4'))` dataset holding `[1.0, 2.0]` reads back as
-  clawhdf5 reads. Reproducer: `h5py.vlen_dtype(np.dtype('>f4'))` dataset holding `[1.0, 2.0]`
+  `[4.6e-41, 9.0e-44]`; `h5dump` prints the file's values. `ref.py` checks that the installed
-  reads back in h5py as `[4.6e-41, 9.0e-44]`. Affected here: `NCAS-CMS_pyfive/tests/data/attr_datatypes.hdf5`, `hdf5/tools/test/testfiles/tcomplex_be.h5`.
+  h5py still does this (by writing and reading exactly that dataset in memory) and, if so,
+  relabels such elements with the file's byte order before hashing, so the values are still
+  compared. Corrected objects: `NCAS-CMS_pyfive/tests/data/attr_datatypes.hdf5` `/@vlen_uint64` (same as clawhdf5), `hdf5/tools/test/testfiles/tcomplex_be.h5` `/VariableLengthDatasetFloatComplex` (same as clawhdf5).
+
+## Other comparison rules
+
 - **Non-IEEE floats and partial-precision integers (N-Bit).** libhdf5 converts a float whose
   bit layout is not IEEE (e.g. `H5Tset_precision` for the N-Bit filter) or an integer with a
   bit offset / reduced precision into the plain numpy type of the same size. The probe
@@ -282,7 +290,6 @@ columns are.
 - 19 x `OSError: Can't synchronously read data (no appropriate function for conversion path)`
 - 1 x `TypeError: unhandled dtype kind M (dtype('…'))`
 - 1 x `TypeError: No NumPy equivalent for TypeTimeID exists`
-- 1 x `KeyError: "…"`
 - 1 x `ValueError: Insufficient precision in available types to represent (N, N, N, N, N)`
 
 ## Reproduce

Cargo.toml

@@ -18,6 +18,7 @@ members = [
     "crates/clawhdf5-bench",
     "crates/clawhdf5-tools",
     "crates/clawhdf5-wasm",
+    "crates/clawhdf5-remote",
     "crates/libaec-sys",
 ]
 resolver = "2"

ROADMAP.md

@@ -1,193 +1,160 @@
-# ClawhDF5 Roadmap — Agent Memory Evolution
+# clawhdf5 roadmap
 
-> Making clawhdf5 the defacto agentic memory solution.
+What has shipped, and what is genuinely next. Everything here is checked
-> Single file. Pure Rust. Zero dependencies. Trusted everywhere.
+against `CHANGELOG.md`, `git log` and [`docs/known-issues.md`](docs/known-issues.md);
+dates are merge dates on `main`. Nothing after v2.7.0 has been released:
+the work since then is on `main` under `CHANGELOG.md` "Unreleased".
+
+_Last updated: 2026-09-28 (at `9b5803f`, PR #21)._
 
 ---
 
-## Track 1: Knowledge Graph in HDF5
+## Done
-**Status:** 🟢 Phase 1 Complete
-**Priority:** Critical
-**Crate:** `clawhdf5-agent`
 
-- [x] **1.1** Entity storage — entities with properties, embeddings, timestamps (created_at/updated_at)
+### Releases
-- [x] **1.2** Relation storage — typed edges with RelationType enum (Temporal/Causal/Associative/Hierarchical/Custom), metadata, timestamps
-- [x] **1.3** Entity extraction helpers — rule-based extraction (Person, Org, Location, Date, Technology, Project) with extract_and_store_entities() integration
-- [x] **1.4** Entity resolution — fuzzy name matching (Levenshtein distance) via resolve_or_create()
-- [x] **1.5** Graph traversal queries — BFS neighbors with depth, subgraph extraction from seeds
-- [x] **1.6** Spreading activation — weighted activation propagation with configurable decay
-- [x] **1.7** Graph-aware retrieval — get_entity_context() for formatted context injection
-- [x] **1.8** Tests — comprehensive tests for all new features
 
-**Research:** Graph-Native Cognitive Memory (2026), Graph-based Agent Memory survey (2026), SYNAPSE (2025)
+| Version | Date | Headline |
+|---|---|---|
+| v2.0.0 | 2026-03-19 | rustyhdf5 (11 crates) and edgehdf5 (4 crates) unified into one workspace as `clawhdf5-*` |
+| v2.1.0 | 2026-06-03 | HNSW backs the agent's vector search by default; live, mutable HNSW index |
+| v2.2.0 – v2.7.0 | 2026-09-18 – 2026-09-20 | bounded decompression and read-path bounds checks, single-writer store locking, WAL v4, HNSW recall fix (0.31 -> 0.98 recall@10 at 100K), fusion weights tuned on LongMemEval, int8 index, Extensible Array read fix and chunk-index checksums |
+
+Details per release: [`CHANGELOG.md`](CHANGELOG.md).
+
+### Since v2.7.0 (unreleased, on `main`)
+
+| PR | Merged | What |
+|---|---|---|
+| #3 | 2026-09-23 | pure-Rust deflate (zlib-rs) by default, no C in the core crates' default build (checked in CI), MSRV 1.92 |
+| #4 | 2026-09-25 | files open in h5py again (every `f32` and every empty dataset clawhdf5 wrote was unreadable by libhdf5); float16 embedding storage |
+| #5 | 2026-09-25 | `HDF5Memory::search` with `SearchOptions` (source filters, re-ranking, confidence); float16 on by default |
+| #6 | 2026-09-25 | `clawhdf5-migrate` writes real agent stores; knowledge-graph fix; dated benchmark re-run |
+| #7 | 2026-09-25 | consolidation benchmark completed (cheaper novelty scoring) |
+| #8 | 2026-09-25 | Ed25519-signed checkpoints (`HDF5Memory::verify`) |
+| #9, #10 | 2026-09-25 | OpenClaw and ZeroClaw integration claims withdrawn — neither ever integrated clawhdf5 |
+| #11 | 2026-09-26 | silent wrong data and libhdf5 interop bugs found by the HDF5 audit fixed |
+| #12 | 2026-09-26 | reproducible conformance sweep over eight public corpora, nightly CI job ([`CONFORMANCE.md`](CONFORMANCE.md)) |
+| #13 | 2026-09-26 | reads HDF5 1.6-era layouts, user blocks, virtual datasets, dense attributes, very large groups |
+| #14 | 2026-09-26 | `h5rs` tools (`ls`, `dump`, `stat`, `diff`, `check`), the browser reader (`clawhdf5-wasm`), libhdf5's header checks, plugin filters (LZF, bitshuffle, bzip2, Blosc), concurrency benchmark |
+| #15 | 2026-09-26 | fast contiguous and concurrent reads, variable-length data, nested groups and links in the writer, Python bindings |
+| #16 | 2026-09-26 | chunked full reads faster than an h5py process pool, writer B-trees of any size, Blosc2 (read), 599/697 conformance |
+| #17 | 2026-09-26 | range reads M0/M1 (indexed name lookups, the `Storage` trait), ZFP (read), in-place editing (`FileEditor`) |
+| #18 | 2026-09-27 | range reads M2/M3 (`File::open_storage`; `clawhdf5-remote`: HTTP(S), S3, GCS, Azure), in-place editing of every chunk index, shrinking, dense attributes |
+| #19 | 2026-09-27 | remote files in the browser (`openUrl`, M4), SWMR reader (`File::open_swmr`, M5), Python remote reads and `'r+'` editing |
+| #20 | 2026-09-28 | benchmarks re-measured: LongMemEval with real MiniLM embeddings, local reads on an idle machine |
+| #21 | 2026-09-28 | remote files open in a few requests (group lookups down the B-tree, `Storage::hint`), `ObjectHeader::parse` back to its earlier speed, the last conformance mismatches resolved: 602/697 ok, 0 mismatch (the run of 2026-09-28 in [`CONFORMANCE.md`](CONFORMANCE.md) still counts 1 our-error, a corrupt N-Bit file libhdf5's own tests refuse) |
+
+### Range reads (design: [`docs/design/range-reads.md`](docs/design/range-reads.md))
+
+- [x] M0 — indexed name lookups (#17)
+- [x] M1 — metadata parsed through the `Storage` trait (#17)
+- [x] M2 — raw data through `Storage`, `File::open_storage` (#18)
+- [x] M3 — `clawhdf5-remote`: HTTP(S) range requests and object stores through a block cache; `h5rs` URLs (#18); Python URLs (#19)
+- [x] M4 — `openUrl` in the browser, restartable "NeedBytes" cache (#19; fewer round trips in #21)
+- [x] M5 — reading files a SWMR writer is appending to ([`docs/design/swmr.md`](docs/design/swmr.md), #19)
+
+### Agent memory (`clawhdf5-agent`)
+
+Shipped before and during the v2 releases, and kept current since:
+knowledge graph with entity extraction and resolution; three-tier
+consolidation with decay; hybrid retrieval (HNSW + BM25, weighted or RRF
+fusion, re-ranking, confidence rejection, query expansion); temporal index
+and session DAG; per-save provenance ledger and write-anomaly detection;
+multi-modal embeddings; WAL with chained CRC32; single-writer locking;
+signed checkpoints. Retrieval is measured, not claimed: see
+[`BENCHMARKS.md`](BENCHMARKS.md) ("LongMemEval Results" reports retrieval
+recall, not QA accuracy; earlier headline numbers that compared different
+granularities were retracted there).
 
 ---
 
-## Track 2: Memory Consolidation Engine
+## Next
-**Status:** 🟢 Phase 1 Complete
-**Priority:** Critical
-**Crate:** `clawhdf5-agent`
 
-- [x] **2.1** Importance scoring — surprise (novelty), correction boost, length scoring with configurable weights
+Not scheduled; listed roughly by how much they unblock. None has a date.
-- [x] **2.2** Three-tier memory model — Working → Episodic → Semantic with bounded capacities
-- [x] **2.3** Time-decay with reactivation — exponential decay with configurable half-life, access resets timestamp
-- [x] **2.4** Bounded memory with graceful degradation — evict lowest-decay entries when over capacity
-- [x] **2.5** Consolidation cycles — promote/evict across tiers based on importance and access thresholds
-- [x] **2.6** Memory statistics — ConsolidationStats with per-tier counts, eviction/promotion tracking
-- [x] **2.7** Tests — comprehensive tests for all features
 
-**Research:** CraniMem (2026), D-MEM (2026), AI Hippocampus survey (2026)
+### Distribution
+
+- [ ] **Publish the crates to crates.io.** Nothing is published; the READMEs
+  say to depend on git. Before publishing: no `publish` settings exist
+  (only `clawhdf5-wasm` has `publish = false`).
+- [ ] **Publish Python wheels to PyPI.** `crates/clawhdf5-py` builds with
+  maturin and is tested in CI, but no wheel is published. The default wheel
+  reads plain `http://` only; `https`/`s3`/`gcs`/`azure` wheels compile C
+  (ring, aws-lc-rs).
+- [ ] **The Node.js package** (`packages/clawhdf5-node` over
+  `clawhdf5-napi`) has never worked and is not in CI: fix it and add CI, or
+  remove it ([known issue](docs/known-issues.md)).
+
+### HDF5 features
+
+- [ ] **SWMR writing.** The reader is done (M5); writing a file while
+  libhdf5 readers follow it is not. Also not covered: remote SWMR (a remote
+  file is pinned at open), `MmapFile`/`LazyFile` SWMR reads, refreshing
+  groups or attributes.
+- [ ] **MPI collective I/O.** `clawhdf5-io`'s `MpiVol` (`mpi-io`) is
+  root-read + broadcast and gather-to-root writes, not collective MPI-IO
+  (`MPI_File_read_at_all`/`write_at_all`).
+- [ ] **Paged-metadata single-request reads.** Files written with paged
+  aggregation (`H5Pset_file_space_strategy(PAGE)`, `h5repack -S PAGE`)
+  keep their metadata in a few pages; range reads could fetch those in one
+  request and use the file's page size as the block size. Today the block
+  size is fixed (1 MiB) and only the first block is read ahead
+  (range-reads design, option (c) as a policy).
+- [ ] **Blosc2 and ZFP encoders.** Both filters are read-only; the other
+  plugin filters (LZF, bitshuffle, bzip2, Blosc 1) read and write.
+- [ ] **External links and external raw data** are explicit errors, not
+  followed.
+- [ ] **Virtual datasets:** the "first missing" view and printf gaps other
+  than 0, source-to-virtual type conversion other than a byte swap, nested
+  virtual sources, source files outside the virtual file's directory.
+- [ ] **Datatypes:** x87 long double and binary128 are refused.
+- [ ] **Writer:** one attribute or link message over 65 515 bytes in dense
+  storage is an error (huge fractal-heap objects); no option to write
+  files HDF5 1.8 can read.
+- [ ] **`FileEditor`:** new chunks in implicit indexes, variable-length and
+  reference data, filters it cannot encode (scale-offset, N-Bit, SZIP),
+  some dense-attribute heap layouts, creating or deleting objects and
+  attributes (also from Python `'r+'`), and no journal (a crash mid-edit
+  can leave the file inconsistent). Freed space is reused only within one
+  editor.
+- [ ] **Selection reads** decode the whole dataset when the selection's
+  bounding box covers more than half of it (a strided `ds[::100]`), and
+  for compact/virtual datasets or a non-default fill value: correct, but
+  more work than needed.
+- [ ] **Readers:** `LazyFile` and `MmapFile` still need the whole file;
+  the zero-copy methods need the file in memory.
+
+### Remote and browser
+
+- [ ] Run the `s3`/`gcs`/`azure` backends against real buckets (only built
+  and URL-parsing-tested so far).
+- [ ] `h5rs` options for request headers and cache settings.
+- [ ] Browser limits in [`docs/known-issues.md`](docs/known-issues.md)
+  ("`clawhdf5-wasm` (browser) limits"): files of 4 GiB or more (wasm32),
+  compound/reference/opaque datasets, round trips per index level. The
+  package doubled in size with `openUrl`
+  ([size table](examples/wasm-viewer/README.md#size)); dropping the
+  function-name section would take a third off the raw size (13% gzipped).
+
+### Quality
+
+- [ ] Scheduled fuzz campaigns: the cargo-fuzz targets
+  ([`crates/clawhdf5-format/fuzz`](crates/clawhdf5-format/fuzz/README.md),
+  and the agent's WAL target) run only by hand or with
+  `CLAWHDF5_FUZZ_SECONDS`.
 
 ---
 
-## Track 3: Hybrid Retrieval Pipeline
+## Withdrawn
-**Status:** 🟢 Phase 1 Complete
-**Priority:** High
-**Crate:** `clawhdf5-agent`
 
-- [x] **3.1** Reciprocal Rank Fusion (RRF) — rrf_hybrid_search() with k=60 constant
+- **OpenClaw integration** (withdrawn 2026-09-25, PR #9). clawhdf5 was
-- [x] **3.2** Multi-factor re-ranking — temporal decay, source authority hierarchy, activation scores (reranker.rs)
+  never an OpenClaw memory plugin; the documented
-- [x] **3.3** Low-confidence rejection — min_score threshold, gap filtering, max_results (confidence.rs)
+  `memory.backend = "clawhdf5"` was never valid. The Rust `ClawhdfBackend`
-- [x] **3.4** Query expansion — synonyms, acronyms, temporal rewrites, morphological variants, knowledge graph aliases + expanded_search() with RRF merge
+  remains as a library API. [`docs/openclaw.md`](docs/openclaw.md) records
-- [x] **3.5** Result explanation — ReRankResult with full score breakdown per factor
+  what a real plugin would need.
-- [x] **3.6** Configurable pipeline — ReRankConfig + ConfidenceConfig with tunable weights/thresholds
+- **ZeroClaw integration** (withdrawn 2026-09-25, PR #10). ZeroClaw has no
-- [x] **3.7** Tests + MemX-comparable benchmarks — 5 integration tests (Hit@1≥90%, search<500ms@100K, BM25<200ms@100K, hybrid<50ms@10K, compact<200ms@10K)
+  clawhdf5 backend, and `clawhdf5-migrate`'s SQLite layout is not
+  ZeroClaw's schema.
 
-**Research:** MemX (2026), SwiftMem (2026)
+The old track-by-track tracker this file used to be (agent-memory
-
+Tracks 1–8, mid-2026) is in git history (`git log -- ROADMAP.md`).
----
-
-## Track 4: Temporal Reasoning
-**Status:** 🟢 Phase 1 Complete
-**Priority:** High
-**Crate:** `clawhdf5-agent`
-
-- [x] **4.1** Temporal index — sorted timestamp index with binary search, insert/remove
-- [x] **4.2** Time-range queries — range_query, before, after, latest, earliest
-- [x] **4.3** Session DAG — parent/child linking, chain walking, time-range overlap queries
-- [x] **4.4** Temporal re-ranking — query hint enum (Latest/Earliest/Around/Between/None) with boost scoring
-- [x] **4.5** Temporal entity tracking — EntityTimeline with state change history + point-in-time reconstruction
-- [x] **4.6** Tests — comprehensive tests for all features
-
-**Research:** MemX temporal gaps (≤43.6% Hit@5), MemoryArena multi-session tasks (2026)
-
----
-
-## Track 5: Memory Security & Provenance
-**Status:** 🟢 Phase 1 Complete
-**Priority:** Medium-High
-**Crate:** `clawhdf5-agent`
-
-- [x] **5.1** Source attribution — MemoryProvenance with source, creator, session, FNV-1a content hash
-- [x] **5.2** Write anomaly detection — rate limiting, 15 injection patterns, source distribution analysis
-- [x] **5.3** Source isolation — per-MemorySource sub-stores preventing cross-contamination
-- [x] **5.4** Memory integrity verification — content hash comparison via verify_integrity()
-- [x] **5.5** Poisoning resistance — pattern detection for prompt injection attempts
-- [x] **5.6** Tests — comprehensive tests including adversarial patterns
-
-**Research:** MemoryGraft (2025), SSGM Framework (2026)
-
----
-
-## Track 6: Multi-Modal Memory
-**Status:** 🟢 Phase 1 Complete
-**Priority:** Medium
-**Crate:** `clawhdf5-agent`
-
-- [x] **6.1** Image embedding storage — ModalEmbedding with model provenance (CLIP, SigLIP, etc.)
-- [x] **6.2** Audio fingerprints — Audio modality with embedding storage
-- [x] **6.3** Multi-modal search — search_by_modality (filtered) + search_cross_modal (all embeddings)
-- [x] **6.4** Observation records — raw perception vs interpretation with confidence scoring
-- [x] **6.5** Media reference storage — MediaRef with Path/Url/Inline, MIME types, FNV-1a checksums
-- [x] **6.6** Tests — 35 comprehensive tests
-
-**Research:** Neuro-Symbolic Memory (2026), RAGdb multi-modal RAG (2025)
-
----
-
-## Track 7: OpenClaw Integration — withdrawn (2026-09-25)
-**Status:** ⚪ Withdrawn (the items below were library work; no OpenClaw integration shipped)
-**Priority:** Critical (for adoption)
-**Crates:** `clawhdf5-agent`, `clawhdf5-napi`
-
-- [x] **7.1** Memory backend trait — MemoryBackend with search/get/write/ingest/export/stats
-- [x] **7.2** Hybrid retrieval pipeline — ClawhdfBackend wires RRF → reranker → confidence rejection
-- [x] **7.3** Markdown import/export — MarkdownParser + MarkdownExporter with line tracking + metadata
-- [x] **7.4** `search()` — backed by the full hybrid retrieval pipeline (a Rust method; no OpenClaw tool was ever registered)
-- [x] **7.5** `get()` — read back by path, with a line slice (not an OpenClaw tool either)
-- [x] **7.6** Compaction integration — run_compaction() (decay + compact + WAL flush), run_consolidation() (hippocampal engine), tick_session(), flush_wal()
-- [ ] **7.7** ~~Config surface — `memory.backend = "clawhdf5"`~~ — never valid OpenClaw config; docs removed
-- [ ] **7.8** ~~Documentation + migration guide~~ — removed: they described an integration that never worked
-
-**Node.js bridge:** `clawhdf5-napi` (napi-rs) and a TypeScript wrapper in `packages/clawhdf5-node` exist but are unpublished, untested in CI and known to be broken (docs/known-issues.md).
-
----
-
-> **Withdrawn.** None of this track produced a working OpenClaw integration: no
-> plugin was built, the documented `memory.backend = "clawhdf5"` config was never
-> valid in any OpenClaw release, and the Node package was never published. The
-> Rust `ClawhdfBackend` remains as a library API. Not pursued for now; see
-> [docs/openclaw.md](docs/openclaw.md) for what a plugin would need today.
-
-## Track 8: Benchmarking & Validation
-**Status:** 🟢 Complete
-**Priority:** High
-**Crates:** `clawhdf5-agent`, `clawhdf5-bench`
-
-- [x] **8.1** MemoryArena benchmark — 35 queries, 50 sessions, Hit@10=91.4%, MRR=0.547
-- [x] **8.2** LongMemEval benchmark — 500 queries, session Hit@1=100%, turn Hit@5=84.4% (beats MemX 51.6%), MRR=0.660
-- [x] **8.3** Latency benchmarks — vector search at 1K/10K/100K, hybrid/RRF, graph traversal, consolidation, temporal
-- [x] **8.4** Memory footprint — 1.7 KB/record uncompressed, 282 B compressed (6.2x ratio), 100K+ rec/s ingestion
-- [x] **8.5** Consolidation efficiency — 8.8x search speedup, 90% noise eviction, zero quality loss
-- [x] **8.6** Cross-platform benchmarks — x86 measured, ARM estimated, cross_platform.sh script
-- [x] **8.7** Published results in BENCHMARKS.md with ephemeral tier Redis comparison (70-140x faster)
-
----
-
-## Implementation Order
-
-**Phase 1:** ~~Tracks 1, 2, 3 — core memory intelligence~~ 🟢 Complete
-**Phase 2:** ~~Track 4 (temporal) + Track 5 (security)~~ 🟢 Complete
-**Phase 3:** ~~Track 6 (multi-modal)~~ 🟢 Complete; Track 7 (OpenClaw integration) withdrawn
-**Phase 4:** ~~Track 8 (benchmarking + validation)~~ 🟢 Complete
-
-All 8 tracks delivered. 1,650+ tests passing, zero clippy warnings.
-
----
-
-## What's Next
-
-Verified against current repo state on 2026-08-05 (see also `docs/superpowers/plans/` for the filter-codec/format-write/MPI-IO work, now shipped):
-
-- [ ] TypeScript bridge not wired into CI — `packages/clawhdf5-node/` already has a complete, working napi-rs package (package.json, tsconfig, hand-written TS wrapper matching all 21 `#[napi]` items, Jest test suite, README); it isn't published to npm and has no committed lockfile
-- [ ] Publish crates to crates.io — no `publish` config anywhere in the workspace yet
-- [ ] Python wheel distribution via maturin — `crates/clawhdf5-py/pyproject.toml` exists (maturin-buildable locally) but wheels aren't published anywhere
-- [ ] `chunked_read.rs`/`data_read.rs` full bounds-check audit + scheduled fuzz campaigns (the new `fuzz_dataset_read` target covers the two files' main entry points; a full manual audit of every indexing site is still open) — see Tier 4 below
-- [ ] WAL per-entry checksum landed as CRC32 (see below); a stronger per-entry format (explicit length prefix, avoiding the read-then-verify restructuring) could still be revisited if profiling shows it matters
-- [ ] HNSW build parallelism is still narrow (only `prune_connections`); the correctness-sensitive outer insert loop needs its own dedicated design pass before parallelizing
-
-### Recently closed out (2026-08-05, Tier 3–4 hardening pass)
-
-- [x] Academic benchmark cross-validation — LongMemEval reproduced against MemX on tank (Ryzen 7 7800X3D): turn-level Hit@5 84.4% vs MemX's 51.6%; recall numbers are deterministic and reproduce exactly across machines. SIMD/Parallelism and Vector Search sections also re-run and dated. See [BENCHMARKS.md § Independent Validation: tank — LongMemEval & Vector Search](BENCHMARKS.md#independent-validation-tank--longmemeval--vector-search-ryzen-7-7800x3d-2026-08-05)
-- [x] Android JNI (`clawhdf5-android`): validate `embedding_len`/`query_embedding_len` against the handle's configured `embedding_dim` before constructing a slice from a raw pointer
-- [x] `clawhdf5-py`: bumped pyo3/numpy 0.28 → 0.29, clearing two RUSTSEC advisories
-- [x] WAL (`clawhdf5-agent`): length-prefix caps (`MAX_WAL_FIELD_LEN`) to reject a corrupted length claim before allocating, then a full per-entry CRC32 trailer (`WAL_VERSION` 2) so a bit-flip stops replay cleanly instead of loading corrupted data; old-format WAL files still read correctly and are migrated on next open
-- [x] `chunked_read.rs`/`data_read.rs`/`local_heap.rs` bounds-check audit: added `ensure_len` overflow guards, a recursion-depth guard against cyclic B-trees, and a fix for an unguarded compound-datatype byte-offset overrun. Added a new `fuzz_dataset_read` cargo-fuzz target exercising the contiguous/chunked/compact read paths — it found and we fixed 3 real crash bugs (integer-overflow panics) within the first few runs
-- [x] `clawhdf5-ann`: optional `parallel` feature (rayon) for HNSW's `prune_connections` neighbor-distance computation
-- [x] `[workspace.dependencies]` added for `tempfile`/`criterion`/`half`/`serde`, fixing a real version skew on `half` (2 vs 2.7)
-
-### Recently closed out (2026-08-05 hardening pass)
-
-- [x] CI/CD pipeline — `.gitea/workflows/ci.yml` now runs `scripts/ci-test.sh` (fmt, clippy, tests, no_std check) on push/PR to `main`
-- [x] Fixed no_std build breakage in `clawhdf5-format` (missing alloc imports, `AtomicU64` unsupported on thumbv7em, `f64::powi` requiring std/libm)
-- [x] Fixed version skew: `clawhdf5-py` (pyproject.toml) and `packages/clawhdf5-node` (package.json) were both behind the actual crate version
-
-### Recently closed out (2026-08-03 cleanup pass)
-
-- [x] Removed `clawhdf5-types` — it was an empty 1-line stub crate; shared type definitions already live in `clawhdf5-format`, so CLAUDE.md and the workspace manifest were corrected instead of filling it in
-- [x] Superblock v4 (page-buffer mode) read/write — the only unimplemented task from `docs/superpowers/plans/2026-06-29-format-write-extensions.md`; now done (`Superblock::parse_v4`/`serialize`, `FileWriter::with_page_size`)
-- [x] Reconciled the three `docs/superpowers/plans/*.md` docs against actual shipped code — they were pre-work plans for `d6c4d4f` (2026-06-30), committed to git late; checkboxes now reflect reality
-
----
-
-_Last updated: 2026-08-05_

benchmarks/cross_platform.sh

@@ -29,7 +29,8 @@
 #     - Use wasm-pack with a custom bench harness
 #     - Replace std::time::Instant with web_sys::Performance::now()
 #     - Replace TempDir/HDF5 I/O with an in-memory backend (separate effort)
-#   See ROADMAP.md §WASM for the full scope.
+#   Browser reads are tested (not benchmarked) by
+#   examples/wasm-viewer/test/run.sh; see examples/wasm-viewer/README.md.
 
 set -euo pipefail
 

conformance/README.md

@@ -6,9 +6,38 @@ h5py/libhdf5, compares the two readings object by object, and writes
 
 ```sh
 CLAWHDF5_PYTHON=/path/to/venv/bin/python conformance/run.sh      # ~30 s once the corpus is cached
+conformance/run.sh --no-fetch                                    # use the cached corpus as is
 conformance/run.sh --update-baseline                             # after an intended change in results
 ```
 
+Latest result (tank, 2026-09-28 04:29 UTC, `conformance/run.sh --no-fetch
+--update-baseline`): 602 of 697 files ok, 1 our-error, 0 mismatch, 2
+ref-bug, 92 h5py-cannot-read, and no panic, hang, crash or out-of-memory.
+The our-error file is `bad_nbit_parms_walk.h5`, which flips between ref-bug
+and our-error from run to run (see `docs/known-issues.md`). The report with every file is
+[`CONFORMANCE.md`](../CONFORMANCE.md).
+
+## Classes
+
+`compare.py` puts each file in one class:
+
+| class | meaning |
+|---|---|
+| **ok** | clawhdf5 and h5py read the same objects with the same values |
+| **our-error** | h5py reads something clawhdf5 refuses |
+| **mismatch** | both read it, with different values or structure |
+| **h5py-cannot-read** | h5py (libhdf5) cannot read the file; not compared |
+| **ref-bug** | h5py reads an object clawhdf5 refuses, but only through a libhdf5 over-read: `ref_bugs.py` re-reads it in six processes with different heaps (import order, `MALLOC_PERTURB_`) and its values change. The file is ref-bug only while that is confirmed in the same run; if the values become stable it counts as our-error again |
+| **panic / hang / crash / oom** | a clawhdf5 failure under the timeout and address-space limit; the gate fails on any |
+
+Where h5py itself returns wrong values through a known h5py bug (the
+big-endian variable-length bug: elements returned with the file's bytes
+under a little-endian dtype), `ref.py` checks that the installed h5py has
+the bug, corrects the values before hashing and marks them `ref_fix`, so
+those objects are still compared. The evidence for the three remaining
+non-ok files (ref-bug or, for one, our-error) is under "Conformance: the last non-ok files" in
+[`docs/known-issues.md`](../docs/known-issues.md).
+
 Needs Rust, `git`, `h5dump` (Debian/Ubuntu `hdf5-tools`), `libaec` (for the
 probe's `szip` feature; `libaec-dev`), and a Python with the packages in
 `requirements.txt`. The first run downloads about 450 MB of sparse checkouts.
@@ -19,9 +48,11 @@ probe's `szip` feature; `libaec-dev`), and a Python with the packages in
 | `fetch-corpus.sh` | shallow, sparse, blob-filtered checkout of each pinned commit into `.cache/src/` (gitignored); no-op when already there |
 | `list_files.py` | which files are probed (HDF5/netCDF-4 extensions minus netCDF classic, plus the CVE reproducers) |
 | `probe/` | the clawhdf5 side: a standalone crate (outside the workspace, so `cargo test --workspace` never builds it) that walks a file with `clawhdf5-format` and prints canonical JSON |
-| `ref.py` | the h5py side: the same JSON from h5py |
+| `ref.py` | the h5py side: the same JSON from h5py (values corrected for a known h5py bug are marked `ref_fix`) |
+| `ref_bugs.py` | re-reads the objects h5py reads only through a libhdf5 bug in six differently-set-up processes; an object whose values change is confirmed as a libhdf5 over-read |
+| `test_ref.py` | tests of `ref.py`'s correction and `ref_bugs.py`'s confirmation (`python conformance/test_ref.py`) |
 | `run_one.sh` | runs both sides on one file (and `h5dump` on the CVE corpus) under a timeout and an address-space limit |
-| `compare.py` | classifies each file (ok / our-error / mismatch / h5py-cannot-read / panic / hang / crash / oom) and groups root causes |
+| `compare.py` | classifies each file (ok / our-error / mismatch / h5py-cannot-read / ref-bug / panic / hang / crash / oom) and groups root causes |
 | `report.py` | writes `CONFORMANCE.md` |
 | `check.py` | the gate: fails on any panic/hang/crash/oom, on an ok count below `baseline.json`, or on a baseline-ok file that is no longer ok |
 | `baseline.json` | the ok files the gate holds the line on |

conformance/baseline.json

@@ -1,35 +1,32 @@
 {
  "comment": "conformance/run.sh fails if the ok count drops below `ok` or a file in `ok_files` stops being ok. Regenerate with `conformance/run.sh --update-baseline` after an intended change.",
- "commit": "73a01f1256fb9bf1b1e7601f755af9e8273cec4e",
+ "commit": "bf5a163dcf7fe28d651ada6545d8136ffeffc825",
- "date": "2026-09-26 14:18 UTC",
+ "date": "2026-09-28 04:29 UTC",
  "reference": "h5py 3.16.0 / HDF5 2.0.0",
  "files": 697,
- "ok": 575,
+ "ok": 602,
  "counts": {
   "h5py-cannot-read": 92,
-  "mismatch": 20,
+  "ok": 602,
-  "ok": 575,
+  "our-error": 1,
-  "our-error": 10
+  "ref-bug": 2
  },
  "per_corpus": {
   "NCAS-CMS_pyfive": {
-   "mismatch": 1,
+   "ok": 33
-   "ok": 32
   },
   "cve_hdf5": {
    "h5py-cannot-read": 32,
-   "mismatch": 9,
+   "ok": 113,
-   "ok": 100,
+   "ref-bug": 2
-   "our-error": 6
   },
   "h5py_data": {
    "ok": 4
   },
   "hdf5": {
    "h5py-cannot-read": 60,
-   "mismatch": 10,
+   "ok": 405,
-   "ok": 392,
+   "our-error": 1
-   "our-error": 4
   },
   "netcdf-c": {
    "ok": 20
@@ -46,6 +43,7 @@
  },
  "ok_files": [
   "NCAS-CMS_pyfive/tests/compact.hdf5",
+  "NCAS-CMS_pyfive/tests/data/attr_datatypes.hdf5",
   "NCAS-CMS_pyfive/tests/data/btreev2.hdf5",
   "NCAS-CMS_pyfive/tests/data/chunked.hdf5",
   "NCAS-CMS_pyfive/tests/data/cmip_bad_eg.nc",
@@ -117,14 +115,22 @@
   "cve_hdf5/cvefiles/cve-2018-17434.h5",
   "cve_hdf5/cvefiles/cve-2018-17435.h5",
   "cve_hdf5/cvefiles/cve-2018-17437.h5",
+  "cve_hdf5/cvefiles/cve-2018-17438",
+  "cve_hdf5/cvefiles/cve-2018-17439",
   "cve_hdf5/cvefiles/cve-2019-8396.h5",
+  "cve_hdf5/cvefiles/cve-2019-8397.h5",
+  "cve_hdf5/cvefiles/cve-2019-8398.h5",
+  "cve_hdf5/cvefiles/cve-2019-9151.h5",
   "cve_hdf5/cvefiles/cve-2019-9152.h5",
   "cve_hdf5/cvefiles/cve-2020-10811.h5",
   "cve_hdf5/cvefiles/cve-2020-18232.h5",
+  "cve_hdf5/cvefiles/cve-2020-18494.h5",
   "cve_hdf5/cvefiles/cve-2021-36977.h5",
   "cve_hdf5/cvefiles/cve-2021-37501.h5",
   "cve_hdf5/cvefiles/cve-2021-45829.h5",
   "cve_hdf5/cvefiles/cve-2021-45833.h5",
+  "cve_hdf5/cvefiles/cve-2021-46243.h5",
+  "cve_hdf5/cvefiles/cve-2021-46244.h5",
   "cve_hdf5/cvefiles/cve-2024-29157.h5",
   "cve_hdf5/cvefiles/cve-2024-29158.h5",
   "cve_hdf5/cvefiles/cve-2024-29159.h5",
@@ -148,18 +154,23 @@
   "cve_hdf5/cvefiles/cve-2024-32615.h5",
   "cve_hdf5/cvefiles/cve-2024-32616.h5",
   "cve_hdf5/cvefiles/cve-2024-32617.h5",
+  "cve_hdf5/cvefiles/cve-2024-32618.h5",
   "cve_hdf5/cvefiles/cve-2024-32619.h5",
   "cve_hdf5/cvefiles/cve-2024-32620.h5",
   "cve_hdf5/cvefiles/cve-2024-32621.h5",
   "cve_hdf5/cvefiles/cve-2024-32622.h5",
+  "cve_hdf5/cvefiles/cve-2024-32623.h5",
   "cve_hdf5/cvefiles/cve-2024-32624.h5",
   "cve_hdf5/cvefiles/cve-2024-33873.h5",
+  "cve_hdf5/cvefiles/cve-2024-33874.h5",
   "cve_hdf5/cvefiles/cve-2024-33875.h5",
   "cve_hdf5/cvefiles/cve-2024-33876.h5",
   "cve_hdf5/cvefiles/cve-2024-33877.h5",
+  "cve_hdf5/cvefiles/cve-2025-2309.h5",
   "cve_hdf5/cvefiles/cve-2025-2310.h5",
   "cve_hdf5/cvefiles/cve-2025-2924.h5",
   "cve_hdf5/cvefiles/cve-2025-2925.h5",
+  "cve_hdf5/cvefiles/cve-2025-44905.h5",
   "cve_hdf5/cvefiles/cve-2025-6269-1.h5",
   "cve_hdf5/cvefiles/cve-2025-6269-2.h5",
   "cve_hdf5/cvefiles/cve-2025-6269-3.h5",
@@ -183,12 +194,14 @@
   "h5py_data/vlen_string_s390x.h5",
   "hdf5/HDF5Examples/C/H5FLT/tfiles/h5ex_d_bitgroom.h5",
   "hdf5/HDF5Examples/C/H5FLT/tfiles/h5ex_d_blosc.h5",
+  "hdf5/HDF5Examples/C/H5FLT/tfiles/h5ex_d_blosc2.h5",
   "hdf5/HDF5Examples/C/H5FLT/tfiles/h5ex_d_bshuf.h5",
   "hdf5/HDF5Examples/C/H5FLT/tfiles/h5ex_d_bzip2.h5",
   "hdf5/HDF5Examples/C/H5FLT/tfiles/h5ex_d_granularbr.h5",
   "hdf5/HDF5Examples/C/H5FLT/tfiles/h5ex_d_jpeg.h5",
   "hdf5/HDF5Examples/C/H5FLT/tfiles/h5ex_d_lz4.h5",
   "hdf5/HDF5Examples/C/H5FLT/tfiles/h5ex_d_lzf.h5",
+  "hdf5/HDF5Examples/C/H5FLT/tfiles/h5ex_d_zfp.h5",
   "hdf5/HDF5Examples/C/H5FLT/tfiles/h5ex_d_zstd.h5",
   "hdf5/HDF5Examples/C/H5G/16/h5ex_g_iterate.h5",
   "hdf5/HDF5Examples/C/H5G/16/h5ex_g_traverse.h5",
@@ -264,6 +277,7 @@
   "hdf5/test/testfiles/tmtimeo.h5",
   "hdf5/test/testfiles/tnullspace.h5",
   "hdf5/test/testfiles/tsizeslheap.h5",
+  "hdf5/tools/test/testfiles/bigendian/tall.h5",
   "hdf5/tools/test/testfiles/bigendian/tdset2.h5",
   "hdf5/tools/test/testfiles/binfp64.h5",
   "hdf5/tools/test/testfiles/binin16.h5",
@@ -284,6 +298,7 @@
   "hdf5/tools/test/testfiles/h5clear_fsm_persist_noclose.h5",
   "hdf5/tools/test/testfiles/h5clear_fsm_persist_user_equal.h5",
   "hdf5/tools/test/testfiles/h5clear_fsm_persist_user_less.h5",
+  "hdf5/tools/test/testfiles/h5clear_mdc_image.h5",
   "hdf5/tools/test/testfiles/h5clear_sec2_v0.h5",
   "hdf5/tools/test/testfiles/h5clear_sec2_v2.h5",
   "hdf5/tools/test/testfiles/h5copy_extlinks_src.h5",
@@ -337,6 +352,7 @@
   "hdf5/tools/test/testfiles/h5diff_softlinks.h5",
   "hdf5/tools/test/testfiles/h5diff_strings1.h5",
   "hdf5/tools/test/testfiles/h5diff_strings2.h5",
+  "hdf5/tools/test/testfiles/h5diff_types.h5",
   "hdf5/tools/test/testfiles/h5fc_edge_v3.h5",
   "hdf5/tools/test/testfiles/h5fc_err_level.h5",
   "hdf5/tools/test/testfiles/h5fc_ext1_f.h5",
@@ -414,9 +430,11 @@
   "hdf5/tools/test/testfiles/tCVE_2018_11206_fill_new.h5",
   "hdf5/tools/test/testfiles/tCVE_2018_11206_fill_old.h5",
   "hdf5/tools/test/testfiles/taindices.h5",
+  "hdf5/tools/test/testfiles/tall.h5",
   "hdf5/tools/test/testfiles/tarray1.h5",
   "hdf5/tools/test/testfiles/tarray1_big.h5",
   "hdf5/tools/test/testfiles/tarray2.h5",
+  "hdf5/tools/test/testfiles/tarray3.h5",
   "hdf5/tools/test/testfiles/tarray4.h5",
   "hdf5/tools/test/testfiles/tarray5.h5",
   "hdf5/tools/test/testfiles/tarray8.h5",
@@ -436,6 +454,7 @@
   "hdf5/tools/test/testfiles/tcmpdints.h5",
   "hdf5/tools/test/testfiles/tcmpdintsize.h5",
   "hdf5/tools/test/testfiles/tcomplex.h5",
+  "hdf5/tools/test/testfiles/tcomplex_be.h5",
   "hdf5/tools/test/testfiles/tcompound.h5",
   "hdf5/tools/test/testfiles/tcompound_complex.h5",
   "hdf5/tools/test/testfiles/tcompound_complex2.h5",
@@ -449,6 +468,7 @@
   "hdf5/tools/test/testfiles/textlinksrc.h5",
   "hdf5/tools/test/testfiles/textlinktar.h5",
   "hdf5/tools/test/testfiles/textpfe.h5",
+  "hdf5/tools/test/testfiles/tfcontents1.h5",
   "hdf5/tools/test/testfiles/tfcontents2.h5",
   "hdf5/tools/test/testfiles/tfilters.h5",
   "hdf5/tools/test/testfiles/tfloat16.h5",
@@ -505,6 +525,7 @@
   "hdf5/tools/test/testfiles/tstr3.h5",
   "hdf5/tools/test/testfiles/tudfilter.h5",
   "hdf5/tools/test/testfiles/tudfilter2.h5",
+  "hdf5/tools/test/testfiles/tudlink.h5",
   "hdf5/tools/test/testfiles/tvldtypes1.h5",
   "hdf5/tools/test/testfiles/tvldtypes2.h5",
   "hdf5/tools/test/testfiles/tvldtypes3.h5",
@@ -513,6 +534,8 @@
   "hdf5/tools/test/testfiles/tvlenstr_array.h5",
   "hdf5/tools/test/testfiles/tvlstr.h5",
   "hdf5/tools/test/testfiles/tvms.h5",
+  "hdf5/tools/test/testfiles/twithub.h5",
+  "hdf5/tools/test/testfiles/twithub513.h5",
   "hdf5/tools/test/testfiles/txtfp32.h5",
   "hdf5/tools/test/testfiles/txtfp64.h5",
   "hdf5/tools/test/testfiles/txtin16.h5",
@@ -558,6 +581,7 @@
   "hdf5/tools/test/testfiles/xml/tenum.h5",
   "hdf5/tools/test/testfiles/xml/test35.nc",
   "hdf5/tools/test/testfiles/xml/tloop2.h5",
+  "hdf5/tools/test/testfiles/xml/tmany.h5",
   "hdf5/tools/test/testfiles/xml/tname-amp.h5",
   "hdf5/tools/test/testfiles/xml/tname-apos.h5",
   "hdf5/tools/test/testfiles/xml/tname-gt.h5",

conformance/compare.py

@@ -5,6 +5,9 @@ Writes <results_dir>/results.csv, results.json and summary.md.
 File classes (first match wins):
   hang, oom, crash, panic       ours: timeout / allocation failure / signal / any panic (caught or not)
   h5py-cannot-read              libhdf5/h5py failed to open the file (or crashed/hung)
+  ref-bug                       every issue is an object we refuse that h5py reads only through a
+                                libhdf5 bug, confirmed in this run by ref_bugs.py (its values
+                                change with the reading process's heap)
   our-error                     we fail to open, list, or read something h5py reads
   mismatch                      we read something with different shape/values, or a different object set
   ok
@@ -19,6 +22,20 @@ import sys
 R = sys.argv[1]
 RUNS = os.path.join(R, "runs")
 
+# Objects ref_bugs.py confirmed in this run: h5py's values for them come from
+# libhdf5 reading memory the file does not determine.
+try:
+    REF_BUGS = {(b["file"], b["object"])
+                for b in json.load(open(os.path.join(R, "ref_bugs.json")))["read_bugs"] if b.get("confirmed")}
+except (OSError, ValueError, KeyError):
+    REF_BUGS = set()
+
+
+def is_ref_bug(rel, issue):
+    """An our-error on reading an object that ref_bugs.py confirmed."""
+    kind, detail = issue[0], issue[1]
+    return kind == "our-error" and any(f == rel and detail.startswith(obj + ": error: ") for f, obj in REF_BUGS)
+
 
 def load(d, name):
     rc_p = os.path.join(d, name + ".rc")
@@ -86,6 +103,8 @@ mismatch_causes = collections.defaultdict(lambda: {"files": set(), "count": 0, "
 panics = []
 ref_only_errors = collections.Counter()
 incomparable = collections.Counter()
+# Values ref.py corrected for a known h5py bug: (file, object, fixes, same as ours)
+ref_fixes = []
 
 
 def add(bucket, key, file, example):
@@ -146,7 +165,13 @@ for rel in files:
             if a.get("kind") != b.get("kind") and "error" not in b and "error" not in a:
                 issues.append(("mismatch", f"{p}: kind {a.get('kind')} vs ours {b.get('kind')}", "kind", b))
                 ok = False
+            # h5py could not open the object at all: it read none of its
+            # attributes or links, so there is nothing to compare ours with
+            # (the object's own error is compared above and below).
+            ref_unopened = a.get("kind") == "unknown" and "error" in a
             for k in ("error", "list_error", "attrs_error"):
+                if ref_unopened and k != "error":
+                    continue
                 if k in b and k not in a:
                     issues.append(("our-error", f"{p}: {k}: {b[k]}", b[k], b))
                     ok = False
@@ -163,8 +188,10 @@ for rel in files:
                 elif a.get("hash") != b.get("hash"):
                     issues.append(("mismatch", f"{p}: values differ (h5py {a.get('dtype')} vs ours {b.get('dtype')})", "values", b | {"ref_head": a.get("head"), "ref_dtype": a.get("dtype")}))
                     ok = False
+                if a.get("ref_fix") and "hash" in b:
+                    ref_fixes.append((rel, p, a["ref_fix"], a.get("hash") == b.get("hash")))
             ra, oa = a.get("attrs") or {}, b.get("attrs") or {}
-            if "attrs_error" not in b and "attrs_error" not in a:
+            if "attrs_error" not in b and "attrs_error" not in a and not ref_unopened:
                 for an in sorted(set(ra) | set(oa)):
                     x, y = ra.get(an), oa.get(an)
                     if x is None:
@@ -181,6 +208,8 @@ for rel in files:
                         issues.append(("mismatch", f"{p}@{an}: attr shape {x.get('shape')} vs ours {y.get('shape')}", "attr-shape", y | {"ref_dtype": x.get("dtype")}))
                     elif x.get("hash") != y.get("hash"):
                         issues.append(("mismatch", f"{p}@{an}: attr values differ (h5py {x.get('dtype')} vs ours {y.get('dtype')})", "attr-values", y | {"ref_head": x.get("head"), "ref_dtype": x.get("dtype")}))
+                    if x.get("ref_fix") and "hash" in y:
+                        ref_fixes.append((rel, f"{p}@{an}", x["ref_fix"], x.get("hash") == y.get("hash")))
             if ok:
                 n_ok += 1
 
@@ -191,6 +220,8 @@ for rel in files:
         cls = "panic"
     elif ref_open_fail:
         cls = "h5py-cannot-read"
+    elif issues and all(is_ref_bug(rel, i) for i in issues):
+        cls = "ref-bug"
     elif ours_open_err:
         cls = "our-error"
         issues.append(("our-error", f"open: {ours_open_err}", ours_open_err, {}))
@@ -208,7 +239,8 @@ for rel in files:
             "caught": [(p, w, m[:2500]) for p, w, m in caught_panics[:3]],
             "n_caught": len(caught_panics),
         })
-    for kind, detail, key, rec in issues:
+    # A ref-bug file's differences are listed with the evidence instead.
+    for kind, detail, key, rec in (issues if cls != "ref-bug" else []):
         if kind == "our-error":
             add(root_causes, norm(key), rel, detail[:300])
         else:
@@ -253,10 +285,11 @@ def ser(b):
 
 
 json.dump({"rows": rows, "issues": issues_by_file, "root_causes": ser(root_causes), "mismatch_causes": ser(mismatch_causes),
-           "panics": panics, "incomparable": incomparable.most_common(), "ref_only_errors": ref_only_errors.most_common()},
+           "panics": panics, "incomparable": incomparable.most_common(), "ref_only_errors": ref_only_errors.most_common(),
+           "ref_fixes": ref_fixes, "ref_bugs_confirmed": sorted(REF_BUGS)},
           open(os.path.join(R, "results.json"), "w"), indent=1)
 
-classes = ["ok", "our-error", "mismatch", "h5py-cannot-read", "hang", "panic", "crash", "oom"]
+classes = ["ok", "our-error", "mismatch", "h5py-cannot-read", "ref-bug", "hang", "panic", "crash", "oom"]
 by_corpus = collections.defaultdict(collections.Counter)
 for r in rows:
     by_corpus[r["corpus"]][r["class"]] += 1

conformance/probe/src/main.rs

@@ -31,7 +31,7 @@ use clawhdf5_format::filter_pipeline::FilterPipeline;
 use clawhdf5_format::group_v1::{self, GroupEntry};
 use clawhdf5_format::group_v2;
 use clawhdf5_format::message_type::MessageType;
-use clawhdf5_format::object_header::ObjectHeader;
+use clawhdf5_format::object_header::{ObjectClass, ObjectHeader};
 use clawhdf5_format::signature;
 use clawhdf5_format::superblock::Superblock;
 use clawhdf5_format::symbol_table::SymbolTableMessage;
@@ -308,18 +308,20 @@ impl<'a> Ctx<'a> {
             )
             .map_err(e)?;
         }
-        let (shape, n) = Self::shape(&ds);
-        rec.insert("shape".into(), shape);
-        if n.saturating_mul(dt.type_size() as u64) > MAX_BYTES {
-            rec.insert("skipped".into(), Value::String("too large".into()));
-            return Ok(());
-        }
         let lm = h
             .messages
             .iter()
             .find(|m| m.msg_type == MessageType::DataLayout)
             .ok_or("MissingMessage(DataLayout)")?;
         let dl = DataLayout::parse(&lm.data, self.os, self.ls).map_err(e)?;
+        // What libhdf5 checks when it opens the dataset (as File::dataset).
+        data_read::check_dataset_storage(&dl, &ds, &dt, self.data.len() as u64).map_err(e)?;
+        let (shape, n) = Self::shape(&ds);
+        rec.insert("shape".into(), shape);
+        if n.saturating_mul(dt.type_size() as u64) > MAX_BYTES {
+            rec.insert("skipped".into(), Value::String("too large".into()));
+            return Ok(());
+        }
         rec.insert(
             "layout".into(),
             Value::String(
@@ -657,6 +659,20 @@ fn is_group(h: &ObjectHeader) -> bool {
     })
 }
 
+/// The probe's kind for an object header: libhdf5's object class
+/// ([`ObjectHeader::object_class`]: group, then dataset — a datatype *and* a
+/// dataspace — then named datatype), which is what h5py opens the object as.
+/// The root group, and a header with only link messages, count as groups.
+fn kind_of(h: &ObjectHeader, is_root: bool) -> &'static str {
+    match h.object_class() {
+        Some(ObjectClass::Group) => "group",
+        Some(ObjectClass::Dataset) => "dataset",
+        _ if is_root || is_group(h) => "group",
+        Some(ObjectClass::NamedDatatype) => "datatype",
+        None => "unknown",
+    }
+}
+
 fn main() {
     install_hook();
     let path = std::env::args().nth(1).expect("usage: probe <file>");
@@ -696,6 +712,42 @@ fn main() {
             return;
         }
     };
+    // libhdf5 decodes the superblock extension at open (an error refuses
+    // the file), and loads a metadata cache image over the file's own
+    // metadata. It loads the image only when it first reads metadata — the
+    // root group — so a file whose image it cannot load still opens and
+    // that read fails. The library decides all three cases with the same
+    // `cache_image_state`: `File` and `MmapFile` open such a file and fail
+    // every object lookup with the image's error, which is what the probe
+    // records here (on the root group, where libhdf5 reports it).
+    use clawhdf5_format::superblock_ext::{self, CacheImageState};
+    let state = match guarded(|| superblock_ext::cache_image_state(hdf5, &sb).map_err(e)) {
+        Ok(x) => x,
+        Err(msg) => {
+            top.insert("open_error".into(), Value::String(msg));
+            println!("{}", Value::Object(top));
+            return;
+        }
+    };
+    let mut image_error = None;
+    let view = match state {
+        CacheImageState::Absent => None,
+        CacheImageState::Unloadable(err) => {
+            image_error = Some(e(err));
+            None
+        }
+        CacheImageState::Loaded(image) => {
+            let mut v = hdf5.to_vec();
+            match image.block(hdf5).and_then(|b| image.apply(b, &mut v)) {
+                Ok(()) => Some(v),
+                Err(err) => {
+                    image_error = Some(e(err));
+                    None
+                }
+            }
+        }
+    };
+    let hdf5: &[u8] = view.as_deref().unwrap_or(hdf5);
     top.insert("superblock_version".into(), json!(sb.version));
     let ctx = Ctx {
         data: hdf5,
@@ -723,6 +775,9 @@ fn main() {
         let mut rec = Map::new();
         rec.insert("path".into(), Value::String(p.clone()));
         let r = guarded(|| {
+            if let Some(msg) = &image_error {
+                return Err(msg.clone());
+            }
             let h = ctx.header(addr)?;
             Ok(h)
         });
@@ -735,23 +790,7 @@ fn main() {
                 continue;
             }
         };
-        let is_ds = h
+        let kind = kind_of(&h, addr == sb.root_group_address);
-            .messages
-            .iter()
-            .any(|m| m.msg_type == MessageType::DataLayout);
-        let kind = if is_ds {
-            "dataset"
-        } else if is_group(&h) || addr == sb.root_group_address {
-            "group"
-        } else if h
-            .messages
-            .iter()
-            .any(|m| m.msg_type == MessageType::Datatype)
-        {
-            "datatype"
-        } else {
-            "unknown"
-        };
         rec.insert("kind".into(), Value::String(kind.into()));
         if kind == "dataset"
             && let Err(msg) = guarded(|| ctx.read_dataset(&h, &mut rec))
@@ -867,6 +906,42 @@ mod tests {
         assert!(ieee_layout(&f32le));
     }
 
+    #[test]
+    fn kind_follows_libhdf5_object_class() {
+        use clawhdf5_format::object_header::HeaderMessage;
+        let header = |types: &[MessageType]| ObjectHeader {
+            version: 2,
+            messages: types
+                .iter()
+                .map(|&msg_type| HeaderMessage {
+                    msg_type,
+                    size: 0,
+                    flags: 0,
+                    creation_order: None,
+                    data: Vec::new(),
+                })
+                .collect(),
+            reference_count: None,
+            flags: 0,
+            access_time: None,
+            modification_time: None,
+            change_time: None,
+            birth_time: None,
+        };
+        use MessageType::*;
+        // cve-2024-33874 `/Dset1`: a datatype and a layout but no dataspace
+        // is a named datatype to libhdf5 (h5py opens it as one).
+        assert_eq!(kind_of(&header(&[Datatype, DataLayout]), false), "datatype");
+        assert_eq!(
+            kind_of(&header(&[Datatype, Dataspace, DataLayout]), false),
+            "dataset"
+        );
+        assert_eq!(kind_of(&header(&[SymbolTable]), false), "group");
+        assert_eq!(kind_of(&header(&[Link]), false), "group");
+        assert_eq!(kind_of(&header(&[]), true), "group");
+        assert_eq!(kind_of(&header(&[]), false), "unknown");
+    }
+
     #[test]
     fn partial_precision_int_is_shifted_and_sign_extended() {
         let dt = Datatype::FixedPoint {

conformance/ref.py

@@ -53,6 +53,49 @@ def packed(dt):
     return dt
 
 
+# --- reference corrections ---------------------------------------------------
+# Where h5py is known to return values the file does not hold, and the right
+# values follow from what it returned, ref.py corrects them and records the
+# correction on the object ("ref_fix"), so the comparison is still a real
+# comparison and CONFORMANCE.md lists every corrected object. Each correction
+# first checks that the installed h5py still has the bug.
+
+# Corrections applied while encoding the current object.
+FIXES = set()
+_BE_VLEN_BUG = None
+
+
+def be_vlen_bug():
+    """h5py (3.16 / HDF5 2.0 at least) returns the elements of a
+    variable-length sequence whose base type is big-endian with the file's
+    big-endian bytes under a native (little-endian) dtype: a
+    `vlen_dtype('>f4')` dataset holding [1.0, 2.0] reads back as
+    [4.6e-41, 9.0e-44]. `h5dump` prints the file's values. Checked once per
+    process by writing and reading exactly that dataset in memory."""
+    global _BE_VLEN_BUG
+    if _BE_VLEN_BUG is None:
+        import io
+        try:
+            bio = io.BytesIO()
+            with h5py.File(bio, "w") as f:
+                d = f.create_dataset("v", (1,), dtype=h5py.vlen_dtype(np.dtype(">f4")))
+                d[0] = np.array([1.0, 2.0], dtype=">f4")
+            with h5py.File(bio, "r") as f:
+                got = np.asarray(f["v"][0])
+            _BE_VLEN_BUG = (got.dtype == np.dtype("<f4")
+                            and got.view(">f4").tolist() == [1.0, 2.0]
+                            and got.tolist() != [1.0, 2.0])
+        except Exception:  # noqa: BLE001
+            _BE_VLEN_BUG = False
+    return _BE_VLEN_BUG
+
+
+def unswapped(got, base):
+    """`got` is `base` (big-endian somewhere) with every field in native
+    little-endian order instead: the shape of h5py's big-endian VL bug."""
+    return base.newbyteorder("<") == got and base != got
+
+
 def canon_el(dt, val, out):
     if dt.fields:
         for n in dt.names:
@@ -79,7 +122,13 @@ def canon_el(dt, val, out):
             base = h5py.check_vlen_dtype(dt)
             if base is None:
                 raise TypeError(f"unhandled object dtype {dt!r}")
-            arr = np.asarray(val if val is not None else [], dtype=base).reshape(-1)
+            arr = np.asarray(val if val is not None else [])
+            if arr.dtype != base and be_vlen_bug() and unswapped(arr.dtype, base):
+                # h5py's big-endian VL bug (see be_vlen_bug): the bytes are
+                # the file's, the dtype label is wrong. Relabel, don't convert.
+                arr = arr.view(base)
+                FIXES.add("h5py-be-vlen")
+            arr = np.asarray(arr, dtype=base).reshape(-1)
             out += b"V" + struct.pack("<I", arr.shape[0])
             if simple(base):
                 out += arr.astype(packed(base)).tobytes()
@@ -111,10 +160,14 @@ def note_conversion(tid, dt, rec):
 
 
 def hash_values(arr, dt, rec):
-    if dt.subdtype is not None:
+    # h5py expands an HDF5 array element type into trailing array dims, a
-        # h5py expands an HDF5 array element type into trailing array dims
+    # nested array type (an array of arrays) into all of them. Converting the
+    # expanded array back to the inner subarray type would broadcast every
+    # element into a whole subarray, so strip every level.
+    while dt.subdtype is not None:
         dt = dt.subdtype[0]
     arr = np.asarray(arr, dtype=dt)
+    FIXES.clear()
     if simple(dt):
         c = np.ascontiguousarray(arr).astype(packed(dt)).tobytes()
     else:
@@ -122,6 +175,8 @@ def hash_values(arr, dt, rec):
         for x in arr.reshape(-1):
             canon_el(dt, x, out)
         c = bytes(out)
+    if FIXES:
+        rec["ref_fix"] = sorted(FIXES)
     rec["hash"] = hashlib.sha256(c).hexdigest()
     rec["head"] = c[:48].hex()
 
@@ -173,9 +228,13 @@ def main(path):
         return
     objects = []
     seen = set()
-    stack = [("/", None)]
+    # Objects h5py cannot open have no ObjectID to deduplicate by; they are
+    # deduplicated by the address their hard link points at instead, as the
+    # probe deduplicates every object by header address.
+    seen_unopenable = set()
+    stack = [("/", None, None)]
     while stack:
-        p, obj = stack.pop()
+        p, obj, link_addr = stack.pop()
         if len(objects) >= MAX_OBJECTS:
             top["truncated"] = True
             break
@@ -185,6 +244,10 @@ def main(path):
                 obj = f[p]
             key = hash(obj.id)  # h5py ObjectID hash = (fileno, object address/token)
         except Exception as e:  # noqa: BLE001
+            if link_addr is not None:
+                if link_addr in seen_unopenable:
+                    continue
+                seen_unopenable.add(link_addr)
             rec["kind"] = "unknown"
             rec["error"] = err(e)
             objects.append(rec)
@@ -232,15 +295,18 @@ def main(path):
                 base = "" if p == "/" else p
                 kids = []
                 for n in names:
+                    # The link's own type: `obj.get(n, getlink=True)` reports
+                    # a user-defined link (type 64-255) as a HardLink.
                     try:
-                        link = obj.get(n, getlink=True)
+                        info = obj.id.links.get_info(n.encode("utf-8", "surrogateescape"))
                     except Exception:  # noqa: BLE001
-                        link = None
+                        info = None
-                    if link is not None and not isinstance(link, h5py.HardLink):
+                    if info is not None and info.type != h5py.h5l.TYPE_HARD:
                         continue
-                    kids.append(f"{base}/{n}")
+                    addr = info.u if info is not None else None
-                for k in reversed(kids):
+                    kids.append((f"{base}/{n}", addr))
-                    stack.append((k, None))
+                for k, addr in reversed(kids):
+                    stack.append((k, None, addr))
             except Exception as e:  # noqa: BLE001
                 rec["list_error"] = err(e)
         objects.append(rec)

conformance/ref_bugs.py

@@ -0,0 +1,105 @@
+#!/usr/bin/env python3
+"""ref_bugs.py <corpus_dir>: re-check the objects h5py reads only through a
+libhdf5 bug.
+
+For each object of READ_BUGS (below), h5py reads it in several fresh
+processes whose heaps differ: h5py imported before numpy (three runs, plus
+two with glibc's MALLOC_PERTURB_, which fills newly allocated and freed heap
+blocks with a byte pattern) and numpy imported first. Values the file
+determines come out the same every time. An object whose values differ
+between those runs is read from memory the file does not determine — an
+over-read or an uninitialised buffer in libhdf5 — so the values h5py reports
+for it are not the file's, and clawhdf5 refusing the object is not a
+clawhdf5 error. compare.py classifies a file as `ref-bug` only on objects
+confirmed that way in the same run (`$OUT/ref_bugs.json`); an object whose
+reading turns out stable stays an our-error.
+
+Run by conformance/run.sh; on its own it is the reproducer (JSON on stdout).
+"""
+import concurrent.futures
+import json
+import os
+import subprocess
+import sys
+
+# (file, object) -> what goes wrong. Checked 2026-09-27 against HDF5 2.0.0
+# (h5py 3.16), h5dump 1.14.6 and the HDFGroup/hdf5 sources (tag hdf5_1_14_6
+# and develop); see docs/known-issues.md, "Conformance: the last non-ok files".
+READ_BUGS = {
+    ("cve_hdf5/cvefiles/cve-2025-2308.h5", "/Scale_offset_long_long_data_le"):
+        "the first chunk records minbits 11: its 12 values need 17 bytes of codes, and the "
+        "26-byte chunk holds 5 after its 21-byte header; libhdf5's scale-offset decoder reads "
+        "past its buffer, and develop refuses the chunk (\"Buffer too short\")",
+    ("cve_hdf5/cvefiles/cve-2025-44904.h5", "/Scale_offset_float_data_le"):
+        "unfiltered chunks stored as 38 and 37 bytes for 48-byte chunks: 1.14/2.0 read the "
+        "stored bytes into a buffer of that size and use it as the whole chunk "
+        "(H5D__chunk_lock), so the rest is heap memory; develop refuses them (\"incorrect chunk "
+        "size returned from index for unfiltered chunk\")",
+    ("hdf5/test/testfiles/bad_nbit_parms_walk.h5", "/Nbit_int_data_le"):
+        "the N-Bit parameter list holds 7 values (cd_values[0] = 7) where an integer needs 8: "
+        "the decoder takes the bit offset from cd_values[7], past the list; libhdf5's own test "
+        "(`test_filter_bad_params`, test/dsets.c on develop) requires the read to fail",
+}
+
+# (which module is imported first, MALLOC_PERTURB_)
+RUNS = [("h5py", None), ("h5py", None), ("h5py", None), ("h5py", "170"), ("h5py", "255"),
+        ("numpy", None)]
+
+READ = r"""
+import hashlib, sys
+if sys.argv[3] == "h5py":
+    import h5py, numpy as np
+else:
+    import numpy as np, h5py
+try:
+    import hdf5plugin  # noqa: F401
+except Exception:
+    pass
+try:
+    with h5py.File(sys.argv[1], "r") as f:
+        a = np.ascontiguousarray(f[sys.argv[2]][()])
+    print("values " + hashlib.sha256(a.tobytes()).hexdigest()[:16])
+except Exception as e:
+    print("error " + (str(e).splitlines() or [type(e).__name__])[0][:120])
+"""
+
+
+def read_once(path, obj, first, perturb):
+    env = dict(os.environ)
+    env.pop("MALLOC_PERTURB_", None)
+    if perturb:
+        env["MALLOC_PERTURB_"] = perturb
+    try:
+        p = subprocess.run([sys.executable, "-c", READ, path, obj, first], env=env,
+                           capture_output=True, text=True, timeout=60)
+        out = p.stdout.strip().splitlines()
+        return out[-1] if out else f"exit {p.returncode}"
+    except subprocess.TimeoutExpired:
+        return "timeout"
+
+
+def check(corpus, key):
+    f, obj = key
+    path = os.path.join(corpus, f)
+    rec = {"file": f, "object": obj, "why": READ_BUGS[key]}
+    if not os.path.exists(path):
+        return rec | {"missing": True, "confirmed": False}
+    runs = [{"first": a, "malloc_perturb": p, "outcome": read_once(path, obj, a, p)} for a, p in RUNS]
+    distinct = sorted({r["outcome"] for r in runs})
+    return rec | {
+        "runs": runs,
+        "distinct": len(distinct),
+        "confirmed": len(distinct) > 1 and any(o.startswith("values ") for o in distinct),
+    }
+
+
+def main():
+    corpus = sys.argv[1]
+    keys = list(READ_BUGS)
+    with concurrent.futures.ThreadPoolExecutor(max_workers=len(keys)) as ex:
+        out = list(ex.map(lambda k: check(corpus, k), keys))
+    print(json.dumps({"read_bugs": out}, indent=1))
+
+
+if __name__ == "__main__":
+    main()

conformance/report.py

@@ -26,7 +26,7 @@ except Exception:  # noqa: BLE001
 R, OUT_MD, CORPUS = sys.argv[1], sys.argv[2], sys.argv[3]
 HERE = os.path.dirname(os.path.abspath(__file__))
 ROOT = os.path.dirname(HERE)
-CLASSES = ["ok", "our-error", "mismatch", "h5py-cannot-read", "panic", "hang", "crash", "oom"]
+CLASSES = ["ok", "our-error", "mismatch", "h5py-cannot-read", "ref-bug", "panic", "hang", "crash", "oom"]
 
 
 def sh(*cmd, cwd=ROOT):
@@ -89,22 +89,15 @@ def ex_list(files, n=3):
     return s + (f" (+{len(files) - n} more)" if len(files) > n else "")
 
 
-# --- known causes that are not clawhdf5 bugs --------------------------------
+# --- reference bugs ---------------------------------------------------------
-def is_h5py_be_vlen(i):
+# ref_bugs.py's re-check of the objects h5py reads only through a libhdf5 bug
-    """h5py returns the elements of a VL sequence of a big-endian base type
+# (compare.py classifies on the confirmed ones), and the objects whose h5py
-    with their file (big-endian) bytes but a native-endian dtype."""
+# values ref.py corrected (compare.py's ref_fixes).
-    return (i["kind"] == "mismatch" and i["key"] in ("values", "attr-values")
+try:
-            and (i.get("ref_dtype") == "object") and (i.get("ours_dtype") or "").startswith("vlen(")
+    ref_bugs = json.load(open(os.path.join(R, "ref_bugs.json")))["read_bugs"]
-            and ">" in (i.get("ours_dtype") or ""))
+except (OSError, ValueError, KeyError):
-
+    ref_bugs = []
-
+ref_fixes = res.get("ref_fixes", [])
-known = collections.defaultdict(list)
-for r in rows:
-    if r["class"] != "mismatch":
-        continue
-    iss = issues.get(r["file"], [])
-    if iss and all(is_h5py_be_vlen(i) for i in iss):
-        known["h5py-be-vlen"].append(r["file"])
 
 
 # --- the CVE corpus: clawhdf5 vs h5dump vs h5py ------------------------------
@@ -219,6 +212,7 @@ w("A file's class is the first that applies:")
 w("")
 w("- **panic / hang / crash / oom** — clawhdf5 panicked (caught per object or not), hit the timeout, died on a signal, or failed an allocation. The CI gate fails on any of these.")
 w("- **h5py-cannot-read** — libhdf5 could not open the file (or itself crashed or hung). Nothing to compare against; most are the deliberately malformed CVE reproducers.")
+w("- **ref-bug** — every difference is an object clawhdf5 refuses that h5py reads only through a libhdf5 bug: the values h5py returns for it change with the reading process's heap, re-checked in every run (see *Reference bugs*).")
 w("- **our-error** — clawhdf5 returned an error for something h5py reads.")
 w("- **mismatch** — both read it, but the shapes, values, object set or attribute set differ.")
 w("- **ok** — every object h5py reads, clawhdf5 reads identically.")
@@ -230,10 +224,12 @@ for c in sorted(by_corpus):
     w(f"| {c} | {sum(cnt.values())} | " + " | ".join(str(cnt.get(k, 0)) for k in CLASSES) + " |")
 w(f"| **all** | **{len(rows)}** | " + " | ".join(f"**{total.get(k, 0)}**" for k in CLASSES) + " |")
 w("")
-n_known = sum(len(v) for v in known.values())
+nonok = total.get("our-error", 0) + total.get("mismatch", 0)
-if n_known:
+w(f"**Our errors and mismatches: {nonok}.** Files not ok: "
-    w(f"{n_known} of the {total.get('mismatch', 0)} mismatches are a known h5py bug, not ours (see *Known not-our-bug*).")
+  + (", ".join(f"{total[c]} {c}" for c in CLASSES if c != "ok" and total.get(c)) or "none") + "."
-    w("")
+  + (f" {len(ref_fixes)} object(s) were compared against h5py's values corrected for a known h5py bug"
+     f" ({sum(1 for x in ref_fixes if x[3])} identical to clawhdf5's; see *Reference bugs*)." if ref_fixes else ""))
+w("")
 w("Corpora (fetched by `conformance/fetch-corpus.sh` into the gitignored `conformance/.cache/`):")
 w("")
 w("| corpus | source | commit |")
@@ -253,19 +249,25 @@ w("")
 
 w("## Our-error root causes")
 w("")
-w("Grouped by normalised error message. *files* counts files whose class this cause affects.")
+if res["root_causes"]:
-w("")
+    w("Grouped by normalised error message. *files* counts files whose class this cause affects.")
-w("| files | objects | error | examples |")
+    w("")
-w("|---:|---:|---|---|")
+    w("| files | objects | error | examples |")
-for k, v in res["root_causes"].items():
+    w("|---:|---:|---|---|")
+    for k, v in res["root_causes"].items():
         w(f"| {v['files']} | {v['count']} | `{k.replace('|', '/')}` | {ex_list(v['file_list'])} |")
+else:
+    w("None.")
 w("")
 w("## Mismatch root causes")
 w("")
-w("| files | objects | cause | examples |")
+if res["mismatch_causes"]:
-w("|---:|---:|---|---|")
+    w("| files | objects | cause | examples |")
-for k, v in res["mismatch_causes"].items():
+    w("|---:|---:|---|---|")
+    for k, v in res["mismatch_causes"].items():
         w(f"| {v['files']} | {v['count']} | `{k.replace('|', '/')}` | {ex_list(v['file_list'])} |")
+else:
+    w("None.")
 w("")
 
 w("## CVE corpus: clawhdf5 vs h5dump vs h5py")
@@ -293,14 +295,38 @@ w("")
 w("</details>")
 w("")
 
-w("## Known not-our-bug")
+w("## Reference bugs")
+w("")
+w("### Objects h5py reads only through a libhdf5 bug (*ref-bug*)")
+w("")
+w("clawhdf5 refuses these objects; h5py 3.16 / HDF5 2.0 returns values for them. `conformance/ref_bugs.py`")
+w("re-reads each with h5py in six fresh processes whose heaps differ (h5py imported before numpy, three")
+w("times and twice more with `MALLOC_PERTURB_`, and numpy imported first). Values the file determines")
+w("come out the same every time; these do not, so they are memory libhdf5 over-reads, not the file's")
+w("data. A file is *ref-bug* only while every one of its differences is such an object confirmed in")
+w("the same run; an object that reads the same every time goes back to *our-error*. Reproducer:")
+w("`python conformance/ref_bugs.py conformance/.cache/corpus` (prints every read's outcome).")
+w("")
+w("| file | object | distinct results in 6 reads | confirmed | what goes wrong |")
+w("|---|---|---:|---|---|")
+for b in ref_bugs:
+    n = "missing" if b.get("missing") else b.get("distinct", "?")
+    w(f"| `{b['file']}` | `{b['object']}` | {n} | {'yes' if b.get('confirmed') else '**no**'} | {b['why']} |")
+w("")
+w("### Values corrected for a known h5py bug")
 w("")
 w("- **h5py big-endian variable-length sequences.** h5py returns the elements of a VL sequence")
 w("  whose base type is big-endian with the file's big-endian bytes but a native (little-endian)")
-w("  numpy dtype, so the values it reports are byte-swapped garbage; `h5dump` prints the values")
+w("  numpy dtype: a `h5py.vlen_dtype(np.dtype('>f4'))` dataset holding `[1.0, 2.0]` reads back as")
-w("  clawhdf5 reads. Reproducer: `h5py.vlen_dtype(np.dtype('>f4'))` dataset holding `[1.0, 2.0]`")
+w("  `[4.6e-41, 9.0e-44]`; `h5dump` prints the file's values. `ref.py` checks that the installed")
-w("  reads back in h5py as `[4.6e-41, 9.0e-44]`. Affected here: "
+w("  h5py still does this (by writing and reading exactly that dataset in memory) and, if so,")
-  + (ex_list(sorted(known["h5py-be-vlen"]), 10) if known["h5py-be-vlen"] else "none") + ".")
+w("  relabels such elements with the file's byte order before hashing, so the values are still")
+w("  compared. Corrected objects: "
+  + (", ".join(f"`{f}` `{p}` ({'same as clawhdf5' if same else '**differs from clawhdf5**'})"
+               for f, p, _, same in ref_fixes) if ref_fixes else "none") + ".")
+w("")
+w("## Other comparison rules")
+w("")
 w("- **Non-IEEE floats and partial-precision integers (N-Bit).** libhdf5 converts a float whose")
 w("  bit layout is not IEEE (e.g. `H5Tset_precision` for the N-Bit filter) or an integer with a")
 w("  bit offset / reduced precision into the plain numpy type of the same size. The probe")

conformance/run.sh

@@ -71,6 +71,8 @@ xargs -a "$OUT/files.txt" -d '\n' -P "$JOBS" -I{} bash -c '
   f="$1"; d="$OUT/runs/${f//\//__}"
   case "$f" in cve_hdf5/*) export WITH_H5DUMP=1 ;; esac
   "$HERE/run_one.sh" "$C/$f" "$d"' _ {} 2>"$OUT/probe.log"
+echo "== re-checking the objects h5py reads only through a libhdf5 bug"
+"$PY" "$HERE/ref_bugs.py" "$C" > "$OUT/ref_bugs.json" 2> "$OUT/ref_bugs.err" || true
 echo "== comparing"
 "$PY" "$HERE/compare.py" "$OUT" >/dev/null
 t2=$(date +%s)

conformance/test_ref.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+"""Tests of the reference side's corrections: `python conformance/test_ref.py`.
+
+- ref.py compares a big-endian VL sequence by the file's values even though
+  h5py returns them byte-swapped (and records that it corrected them);
+- ref_bugs.py confirms an object only when its reads disagree.
+"""
+import json
+import os
+import subprocess
+import sys
+import tempfile
+import unittest
+
+import h5py
+import numpy as np
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, HERE)
+import ref_bugs  # noqa: E402
+
+
+def ref_objects(path):
+    out = subprocess.run([sys.executable, os.path.join(HERE, "ref.py"), path],
+                         capture_output=True, text=True, check=True).stdout
+    return {o["path"]: o for o in json.loads(out)["objects"]}
+
+
+class BigEndianVlen(unittest.TestCase):
+    def test_be_vlen_compared_by_file_values(self):
+        with tempfile.TemporaryDirectory() as d:
+            path = os.path.join(d, "v.h5")
+            with h5py.File(path, "w") as f:
+                for name, order in (("be", ">"), ("le", "<")):
+                    t = np.dtype(order + "f4")
+                    ds = f.create_dataset(name, (2,), dtype=h5py.vlen_dtype(t))
+                    ds[0] = np.array([1.0, 2.0], dtype=t)
+                    ds[1] = np.array([3.0], dtype=t)
+                    u = np.dtype(order + "u8")
+                    f.attrs.create(name, [np.array([1, 2], dtype=u), np.array([42], dtype=u)],
+                                   dtype=h5py.vlen_dtype(u))
+            objs = ref_objects(path)
+        be, le = objs["/be"], objs["/le"]
+        # Same values, so the same canonical hash whatever the file's byte order.
+        self.assertEqual(be["hash"], le["hash"])
+        self.assertEqual(objs["/"]["attrs"]["be"]["hash"], objs["/"]["attrs"]["le"]["hash"])
+        self.assertNotIn("ref_fix", le)
+        # And the correction is recorded wherever h5py needed it.
+        import ref
+        if ref.be_vlen_bug():
+            self.assertEqual(be.get("ref_fix"), ["h5py-be-vlen"])
+            self.assertEqual(objs["/"]["attrs"]["be"].get("ref_fix"), ["h5py-be-vlen"])
+
+
+class RefBugsConfirmation(unittest.TestCase):
+    def run_check(self, outcomes):
+        seq = iter(outcomes)
+        saved = ref_bugs.read_once
+        ref_bugs.read_once = lambda *a: next(seq)
+        try:
+            key = next(iter(ref_bugs.READ_BUGS))
+            with tempfile.TemporaryDirectory() as d:
+                p = os.path.join(d, key[0])
+                os.makedirs(os.path.dirname(p))
+                open(p, "wb").close()
+                return ref_bugs.check(d, key)
+        finally:
+            ref_bugs.read_once = saved
+
+    def test_stable_values_are_not_confirmed(self):
+        r = self.run_check(["values a"] * len(ref_bugs.RUNS))
+        self.assertFalse(r["confirmed"])
+
+    def test_changing_values_are_confirmed(self):
+        r = self.run_check(["values a"] * (len(ref_bugs.RUNS) - 1) + ["values b"])
+        self.assertTrue(r["confirmed"])
+        r = self.run_check(["values a"] * (len(ref_bugs.RUNS) - 1) + ["error filter failed"])
+        self.assertTrue(r["confirmed"])
+
+    def test_errors_only_are_not_confirmed(self):
+        # h5py cannot read it at all: nothing it reads, nothing to excuse.
+        r = self.run_check(["error x"] * (len(ref_bugs.RUNS) - 1) + ["error y"])
+        self.assertFalse(r["confirmed"])
+
+    def test_missing_file_is_not_confirmed(self):
+        key = next(iter(ref_bugs.READ_BUGS))
+        with tempfile.TemporaryDirectory() as d:
+            self.assertFalse(ref_bugs.check(d, key)["confirmed"])
+
+
+if __name__ == "__main__":
+    unittest.main()

crates/clawhdf5-accel/Cargo.toml

@@ -3,7 +3,7 @@ name = "clawhdf5-accel"
 version = "2.7.0"
 edition = "2024"
 rust-version.workspace = true
-description = "SIMD-accelerated operations for rustyhdf5"
+description = "SIMD kernels (AVX2, NEON) used by clawhdf5 — pure Rust"
 license = "MIT"
 repository = "https://git.redclaw.dev/quantumclaw/clawhdf5"
 readme = "README.md"

crates/clawhdf5-accel/README.md

@@ -1,24 +1,62 @@
 # clawhdf5-accel
 
-[![crates.io](https://img.shields.io/crates/v/clawhdf5-accel.svg)](https://crates.io/crates/clawhdf5-accel)
+CPU SIMD kernels for vector search: dot products, cosine similarity, L2
-[![docs.rs](https://docs.rs/clawhdf5-accel/badge.svg)](https://docs.rs/clawhdf5-accel)
+distance, norms and int8 dot products, dispatched at run time to the best
+backend the CPU has, with a portable scalar fallback for every operation.
+[`clawhdf5-ann`](../clawhdf5-ann/README.md) and
+[`clawhdf5-agent`](../clawhdf5-agent/README.md) use it in their distance
+loops; it has nothing to do with HDF5 file I/O.
 
-SIMD-accelerated operations for clawhdf5.
+Not on crates.io yet; depend on it from git:
+
+```toml
+[dependencies]
+clawhdf5-accel = { git = "https://git.redclaw.dev/quantumclaw/clawhdf5" }
+```
+
+## API
+
+```rust
+use clawhdf5_accel::{cosine_similarity, detect_backend, dot_i8, dot_product, l2_distance};
+
+let a = [1.0f32, 2.0, 3.0, 4.0];
+let b = [4.0f32, 3.0, 2.0, 1.0];
+assert_eq!(dot_product(&a, &b), 20.0);
+let _cos = cosine_similarity(&a, &b);
+let _l2 = l2_distance(&a, &b);
+assert_eq!(dot_i8(&[1, -2, 3], &[4, 5, -6]), -24);
+println!("{:?}", detect_backend()); // e.g. Avx2 on x86-64, Neon on aarch64
+```
+
+Also `vector_norm`, `batch_norms`, `batch_cosine`, `batch_cosine_prenorm`,
+`f16_to_f32_batch`, `checksum_fletcher32` and `align_to_cache_line`.
+
+## Backends
+
+`detect_backend()` picks once per process: `Avx512` (with the `avx512`
+feature), `Avx2` (AVX2 + FMA), `Neon` (every aarch64 CPU), or `Scalar`.
+`Sse4` and `WasmSimd128` are reported when detected but run the scalar
+kernels.
+`dot_i8`, used by the agent's quantised (int8) HNSW index, runs on
+AVX2 and on NEON — with the `SDOT` instruction (through inline assembly,
+since the intrinsic is unstable) on cores that have dotprod, such as the
+Raspberry Pi 5, and plain NEON on older ones. At equal recall the int8
+index answers 1.63x the queries per second of the f32 one on x86-64
+(AVX2; 2026-09-20, machine not recorded, not re-run) and 1.18x on a
+Raspberry Pi 5 (2026-09-21) ([`BENCHMARKS.md` § Quantising the index copy](../../BENCHMARKS.md#quantising-the-index-copy-quantized_index)).
+
+The aarch64 code is compiled out on x86, so only the `test-arm64` CI job
+builds and tests it.
 
 ## Features
 
-- AVX2 and NEON SIMD acceleration
+| Feature | Default | What | Builds C |
-- AVX-512 support (`avx512` feature)
+|---|---|---|---|
-- Float16 conversion (`float16` feature)
+| `avx512` | no | AVX-512F kernels | no |
-- CRC32 checksum acceleration
+| `float16` | no | `f16_to_f32_batch` through the `half` crate (a software conversion otherwise) | no |
 
-## Usage
+The half-precision conversion used for stored embeddings is
-
+`clawhdf5_format::float16`, not this crate's.
-```rust
-use clawhdf5_accel::checksum::crc32_simd;
-
-let crc = crc32_simd(&data);
-```
 
 ## License
 

crates/clawhdf5-accel/src/lib.rs

@@ -237,7 +237,10 @@ pub fn f16_to_f32_batch(input: &[u16], output: &mut [f32]) {
     convert::f16_to_f32_batch(input, output);
 }
 
-/// Compute Fletcher-32 checksum.
+/// Compute a textbook Fletcher-32 checksum (both sums start at 0xffff).
+///
+/// This is not HDF5's checksum; the Fletcher-32 I/O filter uses
+/// `clawhdf5_format::checksum::fletcher32`.
 pub fn checksum_fletcher32(data: &[u8]) -> u32 {
     checksum::checksum_fletcher32(data)
 }

crates/clawhdf5-agent/README.md

@@ -1,28 +1,124 @@
 # clawhdf5-agent
 
-[![crates.io](https://img.shields.io/crates/v/clawhdf5-agent.svg)](https://crates.io/crates/clawhdf5-agent)
+Persistent memory for AI agents in a single HDF5 file: text chunks with
-[![docs.rs](https://img.shields.io/docsrs/clawhdf5-agent)](https://docs.rs/clawhdf5-agent)
+embeddings and metadata, hybrid search (HNSW vector search + BM25 keyword
+search, fused), sessions, a knowledge graph, a write-ahead log for crash
+safety, and optionally Ed25519-signed checkpoints. Stores open in h5py like
+any other HDF5 file. Built on [`clawhdf5`](../clawhdf5/README.md),
+[`clawhdf5-ann`](../clawhdf5-ann/README.md) and
+[`clawhdf5-accel`](../clawhdf5-accel/README.md).
 
-HDF5-backed persistent memory store for on-device AI agents.
+It is a library: no agent framework integrates it (OpenClaw and ZeroClaw
+integration claims were withdrawn on 2026-09-25; see
+[`docs/openclaw.md`](../../docs/openclaw.md)). The command-line front end
+is [`clawhdf5-cli`](../clawhdf5-cli/README.md).
 
-Built on [clawhdf5](https://crates.io/crates/clawhdf5), clawhdf5-agent provides a vector-searchable memory backend optimized for edge AI workloads. Store embeddings, text chunks, and metadata in a single HDF5 file with SIMD-accelerated similarity search.
+Not on crates.io yet; depend on it from git:
-
-## Features
-
-- Persistent vector store in HDF5 format
-- Cosine similarity and L2 distance search
-- SIMD-accelerated via clawhdf5-accel (AVX2, NEON)
-- Optional GPU acceleration via clawhdf5-gpu
-- Memory-mapped access for large stores
-- f16 storage support for compact embeddings
-
-## Usage
 
 ```toml
 [dependencies]
-clawhdf5-agent = "2.1.0"
+clawhdf5-agent = { git = "https://git.redclaw.dev/quantumclaw/clawhdf5" }
 ```
 
+## Usage
+
+```rust,no_run
+use std::path::PathBuf;
+use clawhdf5_agent::{AgentMemory, HDF5Memory, MemoryConfig, MemoryEntry, SearchOptions};
+
+let config = MemoryConfig::new(PathBuf::from("agent.h5"), "my-agent", 384);
+let mut mem = HDF5Memory::create(config)?;
+
+mem.save(MemoryEntry {
+    chunk: "The deploy key rotates every Monday.".into(),
+    embedding: vec![0.01; 384], // from your embedding model
+    source_channel: "chat".into(),
+    timestamp: 1_790_000_000.0,
+    session_id: "s1".into(),
+    tags: "ops".into(),
+})?;
+
+let query = vec![0.01f32; 384];
+let hits = mem.search(&query, "deploy key", &SearchOptions::new(5).with_sources(["chat"]));
+for h in &hits {
+    println!("{:.3} {}", h.score, h.chunk);
+}
+mem.flush_wal()?; // checkpoint now; otherwise one is made once the WAL holds more than 500 entries (wal_max_entries)
+# Ok::<(), clawhdf5_agent::MemoryError>(())
+```
+
+## What is in it
+
+- **`HDF5Memory`** — `create`, `open` (single writer: an exclusive lock on
+  `<store>.h5.lock`, a second opener gets `MemoryError::Locked`),
+  `open_read_only` (no lock, never writes). Through the `AgentMemory`
+  trait: `save`, `save_batch`, `delete`, `compact`, `count`, `snapshot`,
+  sessions; also `save_or_update`, `delete_batch`, `flush_wal`.
+- **Search** — `search(query_embedding, text, &SearchOptions)`: optional
+  source-channel filter applied before ranking, vector + BM25 fusion
+  (weighted or RRF), Hebbian activation scaling, optional re-ranking
+  (`reranker::ReRankConfig`) and confidence rejection
+  (`confidence::ConfidenceConfig`). `hybrid_search` and
+  `hybrid_search_with` are thin wrappers. The vector stage uses the HNSW
+  index (`hnsw` feature); its graph is saved to `<store>.h5.ann` at each
+  checkpoint and reloaded on open (rebuilt if stale or damaged).
+- **Storage settings** (`MemoryConfig`, persisted with the store):
+  `float16` embeddings (on by default for new stores; 48% smaller file at
+  100K records, same retrieval on LongMemEval), `quantized_index` (int8
+  copy of the vectors in the index, on by default; re-scored against the
+  exact embeddings), `compression` (off by default), HNSW `m`/`ef`
+  parameters, WAL settings (`wal_enabled`, on by default; `wal_max_entries`,
+  500: the WAL is checkpointed into the `.h5` once it holds more).
+- **WAL** (`wal`) — every write is appended to `<store>.h5.wal` with a
+  chained CRC32 per entry, so a corrupted, reordered or spliced entry stops
+  replay. Recovers from a process crash at any point, including between a
+  checkpoint and the WAL truncate. WAL appends are not fsynced: saves since
+  the last checkpoint can be lost on power failure. An unreadable WAL is
+  quarantined to `<store>.h5.wal.corrupt-<ts>`.
+- **Signed checkpoints** (`signing`) — `set_signing_key` signs a manifest
+  (SHA-256 Merkle tree over records, plus settings, sessions and graph) at
+  every checkpoint; `HDF5Memory::verify(path, &public_key)` checks it and
+  locates edits. WAL entries after the checkpoint are not covered.
+- **Knowledge graph** (`knowledge`, `entity_extract`) — `add_entity`,
+  `add_entity_alias`, `add_relation`, `extract_and_store_entities`,
+  traversal and spreading activation.
+- **Also:** sessions (`session`), temporal index (`temporal`),
+  consolidation tiers (`consolidation`), an in-memory TTL tier
+  (`ephemeral`), multi-modal embeddings (`multimodal`), `AGENTS.md`
+  generation (`agents_md`), query expansion, and a session-scoped
+  provenance ledger and write-anomaly detector on every save
+  (`take_anomaly_alerts`; alerts never block a save, and the source is
+  inferred from `source_channel`, not authenticated).
+- `openclaw::ClawhdfBackend` is `search` with re-ranking and confidence
+  on, plus Markdown import/export. The module name is historical: it is not
+  an OpenClaw plugin.
+
+## Features
+
+| Feature | Default | What | Builds C |
+|---|---|---|---|
+| `hnsw` | yes | HNSW vector index (`clawhdf5-ann`); without it the vector stage is an exact linear cosine scan | no |
+| `parallel` | yes | build the HNSW index on a rayon pool (same graph either way) | no |
+| `float16` | yes | f16 helpers in `vector_search` (`half`). Stores' `MemoryConfig::float16` works without it. | no |
+| `fast-math` | no | `matrixmultiply` batch distances in `strategy` | no |
+| `accelerate` | no | Apple Accelerate BLAS in `strategy` (macOS) | links a system framework |
+| `openblas` | no | OpenBLAS in `strategy` | yes (`openblas-src`) |
+| `gpu` | no | `gpu_search` through [`clawhdf5-gpu`](../clawhdf5-gpu/README.md) (wgpu), used by `strategy`, not by `HDF5Memory::search` | no, but needs GPU drivers |
+| `zstd` | no | Zstd instead of deflate when `MemoryConfig::compression` is on | yes (libzstd) |
+| `async` | no | `async_memory` wrapper on tokio | no |
+
+`--no-default-features --features float16` forces the exact linear scan.
+
+## Measurements and limits
+
+- Search recall and latency, file size, LongMemEval and MemoryArena
+  retrieval numbers: [`BENCHMARKS.md`](../../BENCHMARKS.md), measured with
+  the `clawhdf5-bench` binaries (`search_harness`, `longmemeval_bench`,
+  `footprint_bench`, ...).
+- Known issues and their history: [`docs/known-issues.md`](../../docs/known-issues.md).
+- Migrating a SQLite memory database:
+  [`clawhdf5-migrate`](../clawhdf5-migrate/README.md).
+
 ## License
 
 MIT

crates/clawhdf5-agent/src/store_lock.rs

@@ -20,7 +20,7 @@ const LOCK_RETRY_DELAY: std::time::Duration = std::time::Duration::from_millis(1
 /// never leaves a stale lock behind; the empty lock file itself is harmless).
 #[derive(Debug)]
 pub(crate) struct StoreLock {
-    _file: File,
+    file: File,
 }
 
 impl StoreLock {
@@ -42,7 +42,7 @@ impl StoreLock {
         let mut attempts_left = LOCK_RETRIES;
         loop {
             match file.try_lock() {
-                Ok(()) => return Ok(Self { _file: file }),
+                Ok(()) => return Ok(Self { file }),
                 Err(TryLockError::WouldBlock) if attempts_left > 0 => {
                     attempts_left -= 1;
                     std::thread::sleep(LOCK_RETRY_DELAY);
@@ -60,6 +60,16 @@ impl StoreLock {
     }
 }
 
+impl Drop for StoreLock {
+    /// Unlocks before the file is closed: a process another thread forks
+    /// inherits the descriptor until it execs, and a `flock` lasts while any
+    /// descriptor of the open file does, so closing alone could keep the
+    /// store locked for a moment after the drop (see `FileEditor`'s `Drop`).
+    fn drop(&mut self) {
+        let _ = self.file.unlock();
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

crates/clawhdf5-android/Cargo.toml

@@ -3,7 +3,7 @@ name = "clawhdf5-android"
 version = "2.7.0"
 edition = "2024"
 rust-version.workspace = true
-description = "Android JNI bridge for edgehdf5-memory HDF5 backend"
+description = "Android JNI bindings for clawhdf5 agent memory"
 license = "MIT"
 
 [lib]

crates/clawhdf5-android/README.md

@@ -0,0 +1,44 @@
+# clawhdf5-android
+
+A C ABI over [`clawhdf5-agent`](../clawhdf5-agent/README.md) for Android
+apps: a `cdylib` exporting `extern "C"` functions (`edgehdf5_*`, a name
+kept from the project's earlier "edgehdf5" days) that manage an
+`HDF5Memory` through an opaque handle.
+
+The functions are plain C symbols, not JNI-mangled `Java_...` entry points:
+a Kotlin/Java app calls them through a thin JNI shim or JNA of its own. No
+such shim, Gradle project or AAR is in this repository, and the crate is
+not built for an Android target in CI (only its host-side unit tests run
+with the workspace).
+
+## Functions
+
+| Function | What |
+|---|---|
+| `edgehdf5_create(path, agent_id, embedding_dim)` / `edgehdf5_open(path)` | a handle, or null on failure |
+| `edgehdf5_close(handle)` | drop the store; what is not yet checkpointed stays in its WAL, as with any `HDF5Memory` |
+| `edgehdf5_save(handle, ...)` | save one entry; the embedding length is checked against the store's dimension before the pointer is read |
+| `edgehdf5_delete`, `edgehdf5_count`, `edgehdf5_count_active` | |
+| `edgehdf5_hybrid_search(handle, query, len, text, vector_weight, keyword_weight, max_results, out_indices, out_scores, out_chunks)` | results into caller-provided arrays; returns the number written |
+| `edgehdf5_add_session`, `edgehdf5_get_session_summary` | sessions |
+| `edgehdf5_add_entity`, `edgehdf5_add_relation` | knowledge graph |
+| `edgehdf5_free_string` | free a string this library returned |
+
+Every function is `unsafe`: the caller guarantees valid, NUL-terminated
+strings and correctly sized buffers (see each function's `# Safety`
+section), and serialises access to a handle; separate handles are
+independent.
+
+## Build
+
+```bash
+cargo build --release -p clawhdf5-android   # host build; for a device, add --target aarch64-linux-android with the NDK's linker configured
+```
+
+It depends on `clawhdf5-agent` with **default features off**, so there is
+no HNSW index (the vector stage is an exact linear scan) and no rayon
+pool. No C is compiled.
+
+## License
+
+MIT

crates/clawhdf5-ann/README.md

@@ -1,25 +1,70 @@
 # clawhdf5-ann
 
-[![crates.io](https://img.shields.io/crates/v/clawhdf5-ann.svg)](https://crates.io/crates/clawhdf5-ann)
+An HNSW (Hierarchical Navigable Small World) approximate nearest-neighbour
-[![docs.rs](https://docs.rs/clawhdf5-ann/badge.svg)](https://docs.rs/clawhdf5-ann)
+index in pure Rust, with cosine or L2 distance, optional int8 storage of
+the vectors, deletions, and persistence as an HDF5 file. It is the vector
+stage of [`clawhdf5-agent`](../clawhdf5-agent/README.md)'s search (the
+agent's `hnsw` feature, on by default); distances run on
+[`clawhdf5-accel`](../clawhdf5-accel/README.md)'s SIMD kernels.
 
-HNSW approximate nearest neighbor index stored as HDF5.
+Neighbours are chosen with the HNSW paper's diversity heuristic, not plain
+closest-M (which capped recall on clustered data at 0.31 recall@10 at 100K
+vectors).
 
-## Features
+Not on crates.io yet; depend on it from git:
 
-- Build and query HNSW indexes persisted in HDF5 format
+```toml
-- Pure Rust, no C dependencies
+[dependencies]
-- Efficient similarity search for high-dimensional vectors
+clawhdf5-ann = { git = "https://git.redclaw.dev/quantumclaw/clawhdf5" }
+```
 
 ## Usage
 
 ```rust
-use clawhdf5_ann::HnswIndex;
+use clawhdf5_ann::{DistanceMetric, HnswIndex, Storage};
 
-let index = HnswIndex::from_hdf5("vectors.h5").unwrap();
+let vectors: Vec<Vec<f32>> = (0..500)
-let neighbors = index.search(&query, 10);
+    .map(|i| (0..16).map(|j| ((i * 31 + j * 7) % 97) as f32 / 97.0).collect())
+    .collect();
+
+// m = 16 connections per node, ef_construction = 200
+let mut index = HnswIndex::build_with(&vectors, 16, 200, DistanceMetric::Cosine, Storage::Int8);
+let hits = index.search(&vectors[42], 10, 64); // (id, distance), closest first; ef >= k
+assert!(hits[0].1 < 1e-3); // vector 42 itself (or an identical one)
+
+let id = index.insert(vec![0.5; 16]);
+index.mark_deleted(id);
+
+// Persist as HDF5 (a self-contained file: graph and vectors) and load it back
+let bytes = index.to_hdf5_bytes().unwrap();
+let loaded = HnswIndex::load_from_hdf5(&bytes).unwrap();
+assert_eq!(loaded.len(), index.len());
 ```
 
+- `HnswIndex::build` (L2), `build_with_metric`, `build_with` (metric and
+  storage); `new`/`new_with` plus `insert` for an index built
+  incrementally.
+- `Storage::Int8` keeps each vector as `i8`, a quarter of the memory; it
+  applies to `Cosine` only (an L2 index keeps `Float32`). Distances are then
+  approximate, so a caller that needs exact ranking re-scores the
+  candidates, as the agent does.
+- `mark_deleted`, `is_deleted`, `deleted_count`, `active_len`, `compact`
+  (returns the old-to-new id map).
+- `save_to_hdf5(&mut writer)` / `to_hdf5_bytes` / `load_from_hdf5` store
+  the whole index; `graph_to_bytes` / `from_graph_bytes` store only the
+  graph (with a CRC32) for a caller that keeps the vectors elsewhere — the
+  agent's `<store>.h5.ann` sidecar.
+
+## Features
+
+| Feature | Default | What | Builds C |
+|---|---|---|---|
+| `parallel` | no | build the graph on a rayon pool; the graph is identical with or without it | no |
+
+Recall and speed against exact search, for the index alone and in the
+agent: [`BENCHMARKS.md`](../../BENCHMARKS.md), measured with
+`cargo run --release -p clawhdf5-bench --bin search_harness`.
+
 ## License
 
 MIT

crates/clawhdf5-bench/README.md

@@ -0,0 +1,49 @@
+# clawhdf5-bench
+
+The measurement harnesses behind [`BENCHMARKS.md`](../../BENCHMARKS.md):
+HDF5 read and write speed (against libhdf5 and h5py where noted) and the
+agent store's search, footprint and retrieval quality. Not meant for
+publishing; nothing else in the workspace depends on it. Run everything with
+`--release`, and quote numbers with the machine, date and command, as
+`BENCHMARKS.md` does.
+
+## Binaries
+
+| Binary | Measures |
+|---|---|
+| `read_harness` | full reads vs hyperslab selections of a chunked 2-D dataset (compressed and not) and a contiguous one: does a selection cost scale with the selection or the dataset? (`-- --large` for 512 MB) |
+| `concurrent_read` | decoded read throughput vs threads on one open `File`; `scripts/concurrent_read_h5py.py` runs the same workload with h5py (threads and processes) and `scripts/compare_concurrent_read.py` tabulates both |
+| `search_harness` | HNSW recall@10 vs exact search, QPS and latency per `ef`, and end-to-end `HDF5Memory` ingest/checkpoint/open/search at 1K–100K (`--full`); studies: `--float16-study`, `--options-study`, `--signing-study`, `--ann-only --uniform` |
+| `longmemeval_bench` | LongMemEval retrieval recall (turn and session Hit@k, MRR) — **retrieval, not QA accuracy**. Oracle or full `longmemeval_s` haystack; `--features embeddings` (or `embeddings-cuda`) embeds with MiniLM, otherwise the vector stage is inert and the run is BM25-only |
+| `memory_arena` | a deterministic multi-session retrieval benchmark (BM25-only) |
+| `footprint_bench` | file size and bytes per record at 100–100K records, float16 or `--f32`, WAL on/off, compressed or not |
+| `consolidation_efficiency` | retrieval before and after consolidation on signal + noise records |
+| `ephemeral_perf` | the in-memory ephemeral tier's set/get latency |
+| `mpi_io_bench` | `clawhdf5-io`'s `MpiVol` (root-read + broadcast, not collective I/O); needs `--features mpi-io` and `mpirun` |
+
+```bash
+cargo run --release -p clawhdf5-bench --bin search_harness -- --full
+cargo run --release -p clawhdf5-bench --bin read_harness
+```
+
+## Criterion benches and example
+
+- `cargo bench -p clawhdf5-bench` runs `h5bench_write`, `h5bench_read` and
+  `h5bench_meta` (h5bench-style sequential, chunked, strided and metadata
+  workloads). `--features libhdf5-compare` adds the same workloads through
+  libhdf5 (the `hdf5-metno` crate; needs a system libhdf5 1.14).
+- `examples/worldmodel_sampling.rs`: shuffled per-frame reads of a
+  `(N, H, W, C)` `uint8` dataset, clawhdf5 against h5py on the same file.
+
+## Features
+
+| Feature | What | Builds C |
+|---|---|---|
+| `libhdf5-compare` | libhdf5 variants of the Criterion benches | links the system libhdf5 |
+| `mpi-io` | `mpi_io_bench` | yes (`mpi-sys`; needs an MPI installation) |
+| `embeddings` | MiniLM embeddings for `longmemeval_bench` (candle) | yes (a `cc` build dependency in the candle/tokenizers tree) |
+| `embeddings-cuda` | the same on a CUDA GPU (minutes instead of hours on the full haystack) | yes (CUDA) |
+
+## License
+
+MIT

crates/clawhdf5-bench/src/bin/read_harness.rs

@@ -7,15 +7,19 @@
 //! ```text
 //! cargo run --release -p clawhdf5-bench --bin read_harness
 //! cargo run --release -p clawhdf5-bench --bin read_harness -- --large   # 512 MB
+//! cargo run --release -p clawhdf5-bench --bin read_harness -- --v18      # HDF5 1.8 format
+//! cargo run --release -p clawhdf5-bench --bin read_harness -- --chunk 32 # 32 x 32 chunks
 //! ```
+//!
+//! `--v18` writes the file with `libver_bounds(V18, V18)` (version-1 B-tree
+//! chunk indexes) instead of the default 1.10 format (Fixed Array indexes
+//! here), to compare the two.
 
 use std::time::{Duration, Instant};
 
-use clawhdf5::{File, FileBuilder};
+use clawhdf5::{File, FileBuilder, LibVer};
 use clawhdf5_format::selection::Selection;
 
-const CHUNK: u64 = 256;
-
 struct Layout {
     name: &'static str,
     chunked: bool,
@@ -46,16 +50,19 @@ fn value(row: u64, col: u64) -> f64 {
     (row * 100_003 + col) as f64 * 0.5
 }
 
-fn write_file(path: &std::path::Path, rows: u64, cols: u64) {
+fn write_file(path: &std::path::Path, rows: u64, cols: u64, chunk: u64, v18: bool) {
     let data: Vec<f64> = (0..rows)
         .flat_map(|r| (0..cols).map(move |c| value(r, c)))
         .collect();
     let mut builder = FileBuilder::new();
+    if v18 {
+        builder.libver_bounds(LibVer::V18, LibVer::V18);
+    }
     for (i, layout) in LAYOUTS.iter().enumerate() {
         let ds = builder.create_dataset(&format!("d{i}"));
         ds.with_f64_data(&data).with_shape(&[rows, cols]);
         if layout.chunked {
-            ds.with_chunks(&[CHUNK, CHUNK]);
+            ds.with_chunks(&[chunk, chunk]);
         }
         if layout.deflate {
             ds.with_deflate(4);
@@ -91,7 +98,14 @@ fn slab(start: [u64; 2], count: [u64; 2]) -> Selection {
 }
 
 fn main() {
-    let large = std::env::args().any(|a| a == "--large");
+    let args: Vec<String> = std::env::args().collect();
+    let large = args.iter().any(|a| a == "--large");
+    let v18 = args.iter().any(|a| a == "--v18");
+    let chunk: u64 = args
+        .iter()
+        .position(|a| a == "--chunk")
+        .and_then(|i| args.get(i + 1))
+        .map_or(256, |c| c.parse().expect("--chunk N"));
     let (rows, cols) = if large { (8192, 8192) } else { (4096, 2048) };
     let total_mb = (rows * cols * 8) as f64 / (1 << 20) as f64;
     if cfg!(debug_assertions) {
@@ -100,12 +114,21 @@ fn main() {
 
     let dir = tempfile::TempDir::new().unwrap();
     let path = dir.path().join("read_harness.h5");
-    write_file(&path, rows, cols);
+    let t = Instant::now();
-    let file_mb = std::fs::metadata(&path).unwrap().len() as f64 / (1 << 20) as f64;
+    write_file(&path, rows, cols, chunk, v18);
+    let write_ms = t.elapsed().as_secs_f64() * 1e3;
+    let file_bytes = std::fs::metadata(&path).unwrap().len();
+    let file_mb = file_bytes as f64 / (1 << 20) as f64;
 
     println!("## Read harness");
     println!(
-        "\n{rows} x {cols} f64 ({total_mb:.0} MB per dataset), chunks {CHUNK} x {CHUNK}, file {file_mb:.0} MB\n"
+        "\n{rows} x {cols} f64 ({total_mb:.0} MB per dataset), chunks {chunk} x {chunk}, \
+         format {}, file {file_mb:.0} MB ({file_bytes} bytes), written in {write_ms:.0} ms\n",
+        if v18 {
+            "1.8 (v1 B-tree)"
+        } else {
+            "1.10 (default)"
+        }
     );
 
     // (label, selection, elements selected)

crates/clawhdf5-cli/README.md

@@ -0,0 +1,49 @@
+# clawhdf5-cli
+
+The `clawhdf5` command: create, fill, search and inspect a
+[`clawhdf5-agent`](../clawhdf5-agent/README.md) memory store from the
+shell. Output is JSON. (For general HDF5 files use `h5rs` from
+[`clawhdf5-tools`](../clawhdf5-tools/README.md).)
+
+```bash
+cargo install --path crates/clawhdf5-cli     # installs `clawhdf5`; not on crates.io yet
+# or: cargo run -p clawhdf5-cli -- --help
+```
+
+No C is compiled.
+
+## Commands
+
+The store is `--path FILE` (or `CLAWHDF5_PATH`) before the subcommand.
+
+| Command | What |
+|---|---|
+| `create [--agent-id ID] [--dim N] [--wal] [--f32] [--f32-index]` | a new store (dimension 384 by default); float16 embeddings and an int8 index copy unless `--f32` / `--f32-index`. The WAL is off unless `--wal` (the library's default is on), so each save is checkpointed at once |
+| `save [--json '{...}']` | save one entry, from `--json` or stdin: `{"chunk", "embedding", "source_channel", "timestamp", "session_id", "tags"}` |
+| `search --embedding '[...]' [--query TEXT] [-k N] [--vector-weight W] [--keyword-weight W]` | hybrid search (defaults 5 results, weights 0.7 / 0.3) |
+| `recall INDEX` | one entry by index |
+| `stats` | counts and configuration |
+| `flush-wal` | checkpoint the WAL into the `.h5` |
+| `agents-md [--output FILE]` | generate an `AGENTS.md` from the store |
+| `export` | every entry as JSON lines |
+| `snapshot DEST` | a copy of the store's `.h5` file |
+| `keygen --out FILE` | a new Ed25519 signing key (64 hex characters, created owner-only on Unix) |
+| `verify --public-key HEX_OR_FILE` | check a signed store; exit status 2 if it does not verify |
+
+`recall`, `stats`, `agents-md` and `export` open the store read-only
+(no lock, nothing written), so they work while another process has it
+open. `save`, `search` (which records activation boosts) and `flush-wal`
+open it for writing and take the store's lock. With
+`--signing-key FILE` (or `CLAWHDF5_SIGNING_KEY`) every checkpoint a command
+makes is signed; a signed store refuses to checkpoint without the key.
+
+```bash
+clawhdf5 --path mem.h5 create --agent-id demo --dim 3
+echo '{"chunk":"hello","embedding":[0.1,0.2,0.3],"source_channel":"cli","timestamp":0,"session_id":"s1","tags":""}' \
+  | clawhdf5 --path mem.h5 save
+clawhdf5 --path mem.h5 search --embedding '[0.1,0.2,0.3]' --query hello -k 3
+```
+
+## License
+
+MIT

crates/clawhdf5-derive/Cargo.toml

@@ -3,7 +3,7 @@ name = "clawhdf5-derive"
 version = "2.7.0"
 edition = "2024"
 rust-version.workspace = true
-description = "Derive macros for rustyhdf5 HDF5 traits"
+description = "Derive macro (H5Type) for clawhdf5 compound types"
 license = "MIT"
 repository = "https://git.redclaw.dev/quantumclaw/clawhdf5"
 readme = "README.md"

crates/clawhdf5-derive/README.md

@@ -1,28 +1,50 @@
 # clawhdf5-derive
 
-[![crates.io](https://img.shields.io/crates/v/clawhdf5-derive.svg)](https://crates.io/crates/clawhdf5-derive)
+`#[derive(H5Type)]`: maps a Rust struct with named fields to an HDF5
-[![docs.rs](https://docs.rs/clawhdf5-derive/badge.svg)](https://docs.rs/clawhdf5-derive)
+compound datatype. The derive generates three inherent methods:
 
-Derive macros for clawhdf5 HDF5 traits.
+- `hdf5_datatype() -> clawhdf5_format::datatype::Datatype` — the
+  `Datatype::Compound` (members in field order, packed, little-endian);
+- `to_bytes(&self) -> Vec<u8>` — one element in that layout;
+- `from_bytes(&[u8]) -> Self` — the reverse (panics if the slice is shorter
+  than the compound).
 
-## Features
+Supported field types: `f32`, `f64`, `i8`–`i64`, `u8`–`u64`, `bool`
+(stored as `u8`) and fixed-size arrays `[T; N]` of those numeric types.
+Tuple structs, enums and nested structs are refused at compile time.
 
-- `#[derive(HDF5Type)]` for automatic HDF5 datatype mapping
+The generated code names `clawhdf5_format`, so the crate using the derive
-- Struct-to-compound-type derivation
+must depend on [`clawhdf5-format`](../clawhdf5-format/README.md) too. Not
+on crates.io yet:
 
-## Usage
+```toml
+[dependencies]
+clawhdf5-derive = { git = "https://git.redclaw.dev/quantumclaw/clawhdf5" }
+clawhdf5-format = { git = "https://git.redclaw.dev/quantumclaw/clawhdf5" }
+```
+
+## Example
 
 ```rust
-use clawhdf5_derive::HDF5Type;
+use clawhdf5_derive::H5Type;
+use clawhdf5_format::datatype::Datatype;
 
-#[derive(HDF5Type)]
+#[derive(H5Type, Debug, PartialEq)]
 struct Point {
-    x: f64,
+    id: u32,
-    y: f64,
+    pos: [f64; 3],
-    z: f64,
+    valid: bool,
 }
+
+let p = Point { id: 7, pos: [1.0, 2.0, 3.0], valid: true };
+let bytes = p.to_bytes();
+assert_eq!(bytes.len(), 4 + 24 + 1);
+assert_eq!(Point::from_bytes(&bytes), p);
+assert!(matches!(Point::hdf5_datatype(), Datatype::Compound { size: 29, .. }));
 ```
 
+Tests: `crates/clawhdf5-format/tests/derive_tests.rs`.
+
 ## License
 
 MIT

crates/clawhdf5-filters/README.md

@@ -1,27 +1,56 @@
 # clawhdf5-filters
 
-[![crates.io](https://img.shields.io/crates/v/clawhdf5-filters.svg)](https://crates.io/crates/clawhdf5-filters)
+Standalone deflate (zlib) compression and decompression with a choice of
-[![docs.rs](https://docs.rs/clawhdf5-filters/badge.svg)](https://docs.rs/clawhdf5-filters)
+backend: pure-Rust zlib-rs (default), zlib-ng, Apple's Compression
+framework, or miniz_oxide.
 
-Filter and compression pipeline for clawhdf5.
+This crate holds **deflate backends only**. The HDF5 filter pipeline, the
+filter registry and every other codec (shuffle, Fletcher-32, N-Bit,
+scale-offset, LZ4, Zstd, SZIP, pcodec, LZF, bitshuffle, bzip2, Blosc,
+Blosc2, ZFP) live in [`clawhdf5-format`](../clawhdf5-format/README.md),
+which calls flate2 itself and selects its deflate backend with its own
+features. No library crate of the workspace depends on this one (the
+`clawhdf5` facade uses it only in tests).
 
-## Features
+Not on crates.io yet; depend on it from git:
 
-- DEFLATE compression/decompression
+```toml
-- Pure-Rust deflate via zlib-rs (default, `zlib-rs` feature)
+[dependencies]
-- zlib-ng instead, if you want it (`fast-deflate` feature; C, needs cmake)
+clawhdf5-filters = { git = "https://git.redclaw.dev/quantumclaw/clawhdf5" }
-- Apple Compression framework support (`apple-compression` feature)
+```
 
-## Usage
+## API
 
 ```rust
-use clawhdf5_filters::{deflate_compress, deflate_decompress};
+use clawhdf5_filters::{deflate_backend, deflate_compress, deflate_decompress};
 
+let data: Vec<u8> = (0..10_000u32).map(|i| (i % 251) as u8).collect();
 let compressed = deflate_compress(&data, 6).unwrap();
 // The second argument bounds the output: the expected decompressed size.
 let decompressed = deflate_decompress(&compressed, data.len()).unwrap();
+assert_eq!(decompressed, data);
+println!("backend: {}", deflate_backend()); // "zlib-rs" by default
 ```
 
+Also `deflate_compress_miniz`/`deflate_decompress_miniz` (always
+miniz_oxide) and `fast_deflate::{compress, decompress, active_backend}`.
+
+## Features
+
+Backend priority: `apple-compression` (macOS only) > zlib-ng > zlib-rs >
+miniz_oxide (with none enabled).
+
+| Feature | Default | Backend | Builds C |
+|---|---|---|---|
+| `zlib-rs` | yes | zlib-rs through flate2, with `runtime_detection` (needed for its SIMD) | no |
+| `fast-deflate` | no | zlib-ng through flate2 | yes (cmake) |
+| `system-zlib` | no | the system zlib through flate2 | yes (`libz-sys`) |
+| `apple-compression` | no | Apple Compression framework, macOS only (ignored elsewhere) | no (links a system framework) |
+
+zlib-rs matches zlib-ng on HDF5 reads and writes and produces
+byte-identical output: see "Deflate backend" in
+[`BENCHMARKS.md`](../../BENCHMARKS.md).
+
 ## License
 
 MIT

crates/clawhdf5-filters/src/fast_deflate.rs

@@ -327,24 +327,53 @@ fn inflate_bounded(data: &[u8], size_hint: usize, limit: usize) -> Result<Vec<u8
     }
 }
 
+/// Largest compression output reserved at its worst-case size up front.
+const DEFLATE_EXACT_BOUND: usize = 64 << 20;
+
 /// Compress data using flate2 (zlib-ng, zlib-rs or miniz_oxide; see module docs).
 pub(crate) fn flate2_compress(data: &[u8], level: u32) -> Result<Vec<u8>, String> {
     use flate2::{Compress, Compression, FlushCompress, Status};
 
     // zlib's compressBound, plus the zlib header and trailer.
     let bound = data.len() + (data.len() >> 12) + (data.len() >> 14) + (data.len() >> 25) + 13 + 6;
+    // flate2's Rust backends (zlib-rs, miniz_oxide) zero the whole spare
+    // capacity on each call, so a large input's worst-case bound would be
+    // memory held for nothing (4 GiB for a 4 GiB chunk that deflates to a
+    // few MiB): past 64 MiB the output starts at 1/16 of the bound and
+    // doubles as needed.
+    let first = if bound <= DEFLATE_EXACT_BOUND {
+        bound
+    } else {
+        bound / 16
+    };
     let mut out = Vec::new();
-    out.try_reserve_exact(bound)
+    out.try_reserve_exact(first)
         .map_err(|e| format!("deflate: cannot allocate output: {e}"))?;
 
     let mut deflater = Compress::new(Compression::new(level), true);
     loop {
         let (in_before, out_before) = (deflater.total_in(), deflater.total_out());
+        let rest = &data[in_before as usize..];
+        // zlib takes at most u32::MAX input bytes per call, and `Finish`
+        // ends the stream after the bytes it took: input of 4 GiB or more
+        // was cut at 4 GiB - 1. Finish only once the rest fits one call.
+        let flush = if rest.len() > u32::MAX as usize {
+            FlushCompress::None
+        } else {
+            FlushCompress::Finish
+        };
         let status = deflater
-            .compress_vec(&data[in_before as usize..], &mut out, FlushCompress::Finish)
+            .compress_vec(rest, &mut out, flush)
             .map_err(|e| format!("deflate: {e}"))?;
         match status {
-            Status::StreamEnd => return Ok(out),
+            Status::StreamEnd => {
+                if out.capacity() - out.len() > DEFLATE_EXACT_BOUND {
+                    out.shrink_to_fit();
+                }
+                return Ok(out);
+            }
+            // Out of room (the bound makes it unreachable below
+            // `DEFLATE_EXACT_BOUND`): grow rather than fail.
             Status::Ok | Status::BufError if out.len() == out.capacity() => out
                 .try_reserve(out.capacity().max(4096))
                 .map_err(|e| format!("deflate: cannot allocate output: {e}"))?,

crates/clawhdf5-format/Cargo.toml

@@ -76,8 +76,16 @@ bitshuffle = ["lz4_flex", "ruzstd"]
 bzip2 = ["dep:bzip2", "std"]
 # Blosc 1 (32001) with its BloscLZ, LZ4, Snappy, Zlib and Zstandard codecs.
 blosc = ["lz4_flex", "ruzstd", "snap", "deflate", "std"]
+# Blosc2 (32026), read-only: frames, B2ND arrays, and the Blosc codecs above.
+blosc2 = ["blosc"]
+# ZFP (32013, H5Z-ZFP), read-only: every mode, for int32, int64, float and
+# double fields of 1 to 4 dimensions.
+zfp = []
 # Every plugin filter above.
-plugin-filters = ["lzf", "bitshuffle", "bzip2", "blosc"]
+plugin-filters = ["lzf", "bitshuffle", "bzip2", "blosc", "blosc2", "zfp"]
+# Test instrumentation: per-thread counts of heap objects read (see
+# `lookup_stats`), so tests can bound the cost of a name lookup.
+lookup-stats = ["std"]
 
 [[bench]]
 name = "parallel_decompress_bench"

crates/clawhdf5-format/README.md

@@ -1,27 +1,108 @@
 # clawhdf5-format
 
-[![crates.io](https://img.shields.io/crates/v/clawhdf5-format.svg)](https://crates.io/crates/clawhdf5-format)
+The HDF5 file format in pure Rust: parsers and writers for every on-disk
-[![docs.rs](https://docs.rs/clawhdf5-format/badge.svg)](https://docs.rs/clawhdf5-format)
+structure, the filter pipeline and its codecs, and the shared type
+definitions the other crates use. Most users want the
+[`clawhdf5`](../clawhdf5/README.md) facade, which wraps this crate in an
+h5py-like API; use this one directly for low-level access or in `no_std`
+code.
 
-Pure-Rust HDF5 binary format parsing and writing — no C dependencies.
+Not on crates.io yet; depend on it from git:
+
+```toml
+[dependencies]
+clawhdf5-format = { git = "https://git.redclaw.dev/quantumclaw/clawhdf5" }
+```
+
+## What is in it
+
+- **Parsing:** superblock v0–v3 (`superblock`, with the superblock
+  extension and metadata cache images, `superblock_ext`), object headers v1
+  and v2 (`object_header`), every header message the readers use
+  (`datatype`, `dataspace`, `data_layout` v1–v4 including virtual datasets,
+  `fill_value`, `attribute`, `link_message`, `shared_message`, ...), groups
+  old and new (`group_v1` symbol tables with local heaps, `group_v2` with
+  fractal heaps and v2 B-trees), and every chunk index (v1 B-tree, single
+  chunk, implicit, fixed array, extensible array, v2 B-tree).
+- **Reading data:** `data_read` (contiguous, compact, chunked),
+  `partial_read` and `selection` (hyperslabs and points), `vl_data`
+  (variable-length strings and sequences through the global heap),
+  `chunk_cache`.
+- **Storage:** the `storage::Storage` trait (`read_at`, `read_ranges`,
+  `len`, `hint`) that every read path goes through, so a file can be read
+  from memory, a file handle or a remote backend
+  ([`clawhdf5-remote`](../clawhdf5-remote/README.md)).
+- **Writing:** `file_writer::FileWriter` and the builders in
+  `type_builders` (datasets, groups, attributes, compound and enum types,
+  links, virtual datasets, creation-order tracking); chunk indexes and
+  dense-storage B-trees of any size (`chunked_write`, `btree_v2_write`,
+  `ea_writer`, and the version-1 chunk B-tree of `btree_v1_write`). Output
+  is read by h5py and h5dump; `FileWriter::libver_bounds` (`libver`) picks
+  the format: HDF5 1.10 by default, or one HDF5 1.8 reads.
+- **Filters:** `filter_pipeline` and `filter_registry` (look up by ID; other
+  IDs can be registered at run time with `register_filter`). Built in:
+  deflate, shuffle, Fletcher-32, N-Bit, scale-offset; behind features LZ4,
+  Zstd, SZIP (decode), pcodec, and the plugin filters LZF, bitshuffle,
+  bzip2, Blosc 1 (read and write), Blosc2 and ZFP (read only).
+- **Shared pieces:** `float16` (the one IEEE half-precision conversion the
+  workspace uses), `provenance` (SHA-256 dataset hashes), `checksum`
+  (Jenkins lookup3 for v2+ structures).
+
+## Example
+
+```rust
+use clawhdf5_format::file_writer::{AttrValue, FileWriter};
+use clawhdf5_format::{group_v2, object_header, signature, superblock};
+
+// Write a file to memory
+let mut fw = FileWriter::new();
+fw.create_dataset("data")
+    .with_f64_data(&[1.0, 2.0, 3.0])
+    .with_shape(&[3])
+    .set_attr("unit", AttrValue::String("m/s".into()));
+let bytes = fw.finish().unwrap();
+
+// Parse it back: superblock -> path -> object header
+let (_user_block, file) = signature::split_user_block(&bytes).unwrap();
+let sb = superblock::Superblock::parse(file, 0).unwrap();
+let addr = group_v2::resolve_path_any(file, &sb, "data").unwrap();
+let hdr = object_header::ObjectHeader::parse(file, addr as usize, sb.offset_size, sb.length_size)
+    .unwrap();
+assert!(!hdr.messages.is_empty());
+```
 
 ## Features
 
-- Zero-copy superblock, object header, and B-tree parsing
+| Feature | Default | What | Builds C |
-- Chunked dataset read/write with filter pipelines
+|---|---|---|---|
-- `no_std` support (disable `std` feature)
+| `std` | yes | standard library; without it the crate is `no_std` + `alloc` (CI builds it for `thumbv7em-none-eabihf`) | no |
-- Optional parallel reads via Rayon
+| `checksum` | yes | verify Jenkins lookup3 checksums | no |
-- SHA-256 provenance tracking
+| `deflate` | yes | deflate through flate2 | no |
+| `zlib-rs` | yes | flate2's pure-Rust zlib-rs backend, with `runtime_detection` (without it zlib-rs loses SIMD and inflates 3.5x slower) | no |
+| `system-zlib-decompress` | yes | macOS only: inflate with the system libz first, falling back to flate2; no effect elsewhere | no (links the system libz on macOS) |
+| `provenance` | yes | SHA-256 provenance hashes | no |
+| `lzf` | yes | LZF (32000) | no |
+| `parallel` | no | rayon-parallel chunk decoding | no |
+| `fast-checksum` | no | hardware CRC32 through `crc32fast` | no |
+| `lz4` | no | LZ4 (32004) | no |
+| `pcodec` | no | pcodec | no |
+| `bitshuffle`, `bzip2`, `blosc` | no | 32008, 307, 32001, read and write | no |
+| `blosc2`, `zfp` | no | 32026, 32013, read only | no |
+| `plugin-filters` | no | all six plugin filters above | no |
+| `lookup-stats` | no | counters for name-lookup benchmarks | no |
+| `zstd` | no | Zstandard (32015) | yes (libzstd) |
+| `szip` | no | SZIP (4) decoding | links the system libaec (`libaec-dev`) |
+| `fast-deflate` | no | zlib-ng | yes (cmake) |
+| `system-zlib` | no | the system zlib | yes (`libz-sys`) |
+| `blake3_hash` | no | `provenance::blake3_hash` | yes (`cc`) |
 
-## Usage
+## Robustness
 
-```rust
+Every parser is meant to return an error, never panic, on hostile input:
-use clawhdf5_format::Superblock;
+nine cargo-fuzz targets live in [`fuzz/`](fuzz/README.md), the conformance
-
+sweep includes the HDF Group's CVE corpus
-let data = std::fs::read("data.h5").unwrap();
+([`CONFORMANCE.md`](../../CONFORMANCE.md)), and header checks follow
-let sb = Superblock::from_bytes(&data).unwrap();
+libhdf5's. Open gaps are in [`docs/known-issues.md`](../../docs/known-issues.md).
-println!("HDF5 version {}.{}", sb.version_major(), sb.version_minor());
-```
 
 ## License
 

crates/clawhdf5-format/fuzz/README.md

@@ -51,10 +51,18 @@ done
 
 ## CI
 
-These targets are **not** run in CI (`.gitea/workflows/ci.yml`) — cargo-fuzz
+These targets are **not** run by the CI workflows (`.gitea/workflows/ci.yml`)
-requires nightly and each meaningful run takes minutes, which doesn't fit a
+— cargo-fuzz requires nightly and each meaningful run takes minutes, which
-per-PR gate. Run them manually on a schedule (e.g. before a release, or after
+doesn't fit a per-PR gate. Run them by hand before a release or after
-touching parser code) instead.
+touching parser code. `scripts/ci-test.sh` has an opt-in smoke run: with
+`CLAWHDF5_FUZZ_SECONDS=N` it runs every target of this crate and of
+`crates/clawhdf5-agent/fuzz` (the WAL parser) for N seconds each.
+
+Other robustness checks that do run: the nightly conformance sweep reads
+the HDF Group's CVE reproducers and fails on any panic, hang, crash or
+out-of-memory ([`conformance/README.md`](../../../conformance/README.md)),
+and `scripts/h5rs-fuzz.sh` runs every `h5rs` subcommand over them, optionally
+on byte-flipped copies.
 
 ## Reproducing Crashes
 

crates/clawhdf5-format/src/addr.rs

@@ -0,0 +1,121 @@
+//! File address and length → in-memory index conversion.
+//!
+//! HDF5 addresses and lengths are 64-bit; the file is parsed through a
+//! `&[u8]` indexed by `usize`. On a 64-bit target every `u64` fits, but on a
+//! 32-bit one (`wasm32`, `i686`, `thumbv7em`) an address past `usize::MAX`
+//! used to be truncated by an `as usize` cast — silently pointing at another
+//! part of the file — or to panic. [`to_usize`] is the one conversion the
+//! parsers use instead: such an address is a clean
+//! [`FormatError::Overflow`]. It cannot be inside the data anyway: no slice
+//! is longer than `isize::MAX` bytes.
+
+#[cfg(not(feature = "std"))]
+use alloc::format;
+
+use crate::error::FormatError;
+
+/// A file address, offset or length from the file as a `usize` index.
+///
+/// Fails with [`FormatError::Overflow`] when the value does not fit this
+/// platform's `usize` (only possible on targets narrower than 64 bits).
+#[inline]
+pub fn to_usize(value: u64) -> Result<usize, FormatError> {
+    to_index::<usize>(value)
+}
+
+/// A file address for a [`crate::storage::Storage`] read, checked as
+/// [`to_usize`] checks it: the parsers read through 64-bit offsets, but an
+/// address that could not index an in-memory file on this platform is the
+/// same [`FormatError::Overflow`] the slice parsers gave for it.
+#[inline]
+pub fn checked_addr(value: u64) -> Result<u64, FormatError> {
+    to_usize(value).map(|_| value)
+}
+
+/// [`to_usize`] for an index type of any width. `usize` is 64 bits wide on
+/// the hosts CI tests on, where the error path cannot be reached through
+/// `usize`; tests run the same code with `u32` in its place, as on a 32-bit
+/// target.
+#[inline]
+fn to_index<T: TryFrom<u64>>(value: u64) -> Result<T, FormatError> {
+    T::try_from(value).map_err(|_| too_large(value))
+}
+
+/// A count or offset into an in-memory buffer (a codec's progress counter,
+/// a size the writer computed from data it holds) as a `usize`, saturating
+/// at `usize::MAX` instead of truncating.
+///
+/// For values that are bounded by the length of something in memory, so
+/// always fit; if one ever did not, a saturated index fails its bounds check
+/// or allocation instead of silently addressing the wrong bytes. A value
+/// read from the file uses [`to_usize`].
+#[inline]
+pub fn saturating_usize(value: u64) -> usize {
+    saturating_index(value, usize::MAX)
+}
+
+/// [`saturating_usize`] for an index type of any width, whose largest
+/// value is `max` (see [`to_index`]).
+#[inline]
+fn saturating_index<T: TryFrom<u64>>(value: u64, max: T) -> T {
+    T::try_from(value).unwrap_or(max)
+}
+
+#[cold]
+#[inline(never)]
+fn too_large(value: u64) -> FormatError {
+    FormatError::Overflow(format!(
+        "file address or length {value:#x} exceeds this platform's address space"
+    ))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn values_that_fit_convert_exactly() {
+        assert_eq!(to_usize(0), Ok(0));
+        assert_eq!(to_usize(0x1234), Ok(0x1234));
+        assert_eq!(to_usize(usize::MAX as u64), Ok(usize::MAX));
+    }
+
+    #[test]
+    fn saturating_conversion_never_wraps() {
+        assert_eq!(saturating_usize(0), 0);
+        assert_eq!(saturating_usize(0x1234), 0x1234);
+        assert_eq!(saturating_usize(usize::MAX as u64), usize::MAX);
+        // Past usize::MAX (32-bit targets) or at u64::MAX: saturates.
+        assert_eq!(saturating_usize(u64::MAX), usize::MAX);
+    }
+
+    #[test]
+    fn values_past_usize_max_are_an_error_not_truncated() {
+        // Reachable through `usize` only where it is narrower than u64 (no
+        // such target runs tests in CI), so the same conversion is run with
+        // u32 standing in for a 32-bit usize.
+        let max = u64::from(u32::MAX);
+        assert_eq!(to_index::<u32>(max), Ok(u32::MAX));
+        for past in [max + 1, max + 0x10, 0x1_0000_1234, u64::MAX] {
+            let err = to_index::<u32>(past).unwrap_err();
+            assert!(
+                matches!(err, FormatError::Overflow(_)),
+                "{past:#x}: {err:?}"
+            );
+        }
+        // Where an `as` cast would have wrapped to a small, valid-looking
+        // index, it is not returned.
+        assert_eq!(0x1_0000_1234_u64 as u32, 0x1234);
+        assert!(to_index::<u32>(0x1_0000_1234).is_err());
+
+        assert_eq!(saturating_index(max + 1, u32::MAX), u32::MAX);
+        assert_eq!(saturating_index(0x1_0000_1234, u32::MAX), u32::MAX);
+        assert_eq!(saturating_index(0x1234, u32::MAX), 0x1234);
+
+        // And through `usize` itself, whichever width it has here.
+        match (usize::MAX as u64).checked_add(1) {
+            Some(past) => assert!(matches!(to_usize(past), Err(FormatError::Overflow(_)))),
+            None => assert_eq!(to_usize(u64::MAX), Ok(usize::MAX)),
+        }
+    }
+}

crates/clawhdf5-format/src/attribute.rs

@@ -5,8 +5,10 @@ use alloc::{borrow::Cow, string::String, vec::Vec};
 #[cfg(feature = "std")]
 use std::borrow::Cow;
 
+use crate::addr::to_usize;
 use crate::attribute_info::AttributeInfoMessage;
-use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records};
+use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records_in, find_btree_v2_records_in};
+use crate::checksum::jenkins_lookup3;
 use crate::data_read;
 use crate::dataspace::Dataspace;
 use crate::datatype::Datatype;
@@ -15,6 +17,7 @@ use crate::fractal_heap::FractalHeapHeader;
 use crate::message_type::MessageType;
 use crate::object_header::ObjectHeader;
 use crate::shared_message;
+use crate::storage::Storage;
 use crate::vl_data;
 
 /// A parsed HDF5 attribute message.
@@ -50,7 +53,7 @@ impl AttributeMessage {
     ///
     /// `length_size` is needed for dataspace dimension parsing.
     pub fn parse(data: &[u8], length_size: u8) -> Result<AttributeMessage, FormatError> {
-        Self::parse_impl(data, length_size, None)
+        Self::parse_impl(data, length_size, None::<(&[u8], u8)>)
     }
 
     /// [`AttributeMessage::parse`] with access to the rest of the file, which
@@ -65,13 +68,24 @@ impl AttributeMessage {
         offset_size: u8,
         length_size: u8,
     ) -> Result<AttributeMessage, FormatError> {
-        Self::parse_impl(data, length_size, Some((file_data, offset_size)))
+        Self::parse_in_storage(data, file_data, offset_size, length_size)
     }
 
-    fn parse_impl(
+    /// [`AttributeMessage::parse_in_file`] with the file behind any
+    /// [`Storage`].
+    pub fn parse_in_storage<S: Storage + ?Sized>(
+        data: &[u8],
+        file: &S,
+        offset_size: u8,
+        length_size: u8,
+    ) -> Result<AttributeMessage, FormatError> {
+        Self::parse_impl(data, length_size, Some((file, offset_size)))
+    }
+
+    fn parse_impl<S: Storage + ?Sized>(
         data: &[u8],
         length_size: u8,
-        file: Option<(&[u8], u8)>,
+        file: Option<(&S, u8)>,
     ) -> Result<AttributeMessage, FormatError> {
         ensure_len(data, 0, 2)?;
         let version = data[0];
@@ -86,19 +100,19 @@ impl AttributeMessage {
 
     /// The bytes of an embedded datatype/dataspace message, following the
     /// shared-message reference when `shared` is set.
-    fn embedded_message<'a>(
+    fn embedded_message<'a, S: Storage + ?Sized>(
         bytes: &'a [u8],
         shared: bool,
         msg_type: MessageType,
         length_size: u8,
-        file: Option<(&[u8], u8)>,
+        file: Option<(&S, u8)>,
     ) -> Result<Cow<'a, [u8]>, FormatError> {
         if !shared {
             return Ok(Cow::Borrowed(bytes));
         }
         let (file_data, offset_size) = file.ok_or(FormatError::UnresolvedSharedMessage)?;
         let shared_ref = shared_message::parse_shared_ref_sized(bytes, offset_size, length_size)?;
-        shared_message::resolve_shared_message(
+        shared_message::resolve_shared_message_in(
             file_data,
             &shared_ref,
             msg_type,
@@ -143,10 +157,10 @@ impl AttributeMessage {
         })
     }
 
-    fn parse_v2(
+    fn parse_v2<S: Storage + ?Sized>(
         data: &[u8],
         length_size: u8,
-        file: Option<(&[u8], u8)>,
+        file: Option<(&S, u8)>,
     ) -> Result<AttributeMessage, FormatError> {
         // Flags: bit 0 = datatype is shared, bit 1 = dataspace is shared.
         let flags = data.get(1).copied().unwrap_or(0);
@@ -197,10 +211,10 @@ impl AttributeMessage {
         })
     }
 
-    fn parse_v3(
+    fn parse_v3<S: Storage + ?Sized>(
         data: &[u8],
         length_size: u8,
-        file: Option<(&[u8], u8)>,
+        file: Option<(&S, u8)>,
     ) -> Result<AttributeMessage, FormatError> {
         // Flags: bit 0 = datatype is shared, bit 1 = dataspace is shared.
         let flags = data.get(1).copied().unwrap_or(0);
@@ -322,9 +336,19 @@ impl AttributeMessage {
         file_data: &[u8],
         offset_size: u8,
         length_size: u8,
+    ) -> Result<Vec<String>, FormatError> {
+        self.read_vl_strings_in(file_data, offset_size, length_size)
+    }
+
+    /// [`Self::read_vl_strings`] over any [`Storage`].
+    pub fn read_vl_strings_in<S: Storage + ?Sized>(
+        &self,
+        file_data: &S,
+        offset_size: u8,
+        length_size: u8,
     ) -> Result<Vec<String>, FormatError> {
         let num_elements = self.dataspace.num_elements();
-        vl_data::read_vl_strings(
+        vl_data::read_vl_strings_in(
             file_data,
             &self.raw_data,
             num_elements,
@@ -341,7 +365,8 @@ fn compute_raw_data(
     dataspace: &Dataspace,
     datatype: &Datatype,
 ) -> Vec<u8> {
-    let num_elements = dataspace.num_elements() as usize;
+    // Saturating, like the product: the size is capped at what is there.
+    let num_elements = usize::try_from(dataspace.num_elements()).unwrap_or(usize::MAX);
     let elem_size = datatype.type_size() as usize;
     let expected_size = num_elements.saturating_mul(elem_size);
     let available = data.len().saturating_sub(pos);
@@ -415,7 +440,20 @@ pub fn extract_attributes_full(
     offset_size: u8,
     length_size: u8,
 ) -> Result<Vec<AttributeMessage>, FormatError> {
-    extract_attributes_with(file_data, header, offset_size, length_size, &mut Err)
+    extract_attributes_full_in(file_data, header, offset_size, length_size)
+}
+
+/// [`extract_attributes_full`] over any [`Storage`]. Dense attribute
+/// storage is indexed by a v2 B-tree, which is not read over [`Storage`]
+/// yet: on a backend without the whole file in memory an object with dense
+/// attributes is [`FormatError::ContiguousStorageRequired`].
+pub fn extract_attributes_full_in<S: Storage + ?Sized>(
+    file: &S,
+    header: &ObjectHeader,
+    offset_size: u8,
+    length_size: u8,
+) -> Result<Vec<AttributeMessage>, FormatError> {
+    extract_attributes_with(file, header, offset_size, length_size, &mut Err)
 }
 
 /// Like [`extract_attributes_full`], but an attribute that cannot be read
@@ -431,6 +469,32 @@ pub fn extract_attributes_tolerant(
     header: &ObjectHeader,
     offset_size: u8,
     length_size: u8,
+) -> Result<(Vec<AttributeMessage>, Vec<FormatError>), FormatError> {
+    extract_attributes_tolerant_core(file_data, header, offset_size, length_size)
+}
+
+/// [`extract_attributes_tolerant`] over any [`Storage`] (see
+/// [`extract_attributes_full_in`] for dense storage). One with the whole
+/// file in memory is read as the slice, by code compiled in this crate (see
+/// [`crate::storage`], "Slice entry points").
+#[inline]
+pub fn extract_attributes_tolerant_in<S: Storage + ?Sized>(
+    file_data: &S,
+    header: &ObjectHeader,
+    offset_size: u8,
+    length_size: u8,
+) -> Result<(Vec<AttributeMessage>, Vec<FormatError>), FormatError> {
+    match file_data.as_contiguous() {
+        Some(all) => extract_attributes_tolerant(all, header, offset_size, length_size),
+        None => extract_attributes_tolerant_core(file_data, header, offset_size, length_size),
+    }
+}
+
+fn extract_attributes_tolerant_core<S: Storage + ?Sized>(
+    file_data: &S,
+    header: &ObjectHeader,
+    offset_size: u8,
+    length_size: u8,
 ) -> Result<(Vec<AttributeMessage>, Vec<FormatError>), FormatError> {
     let mut errors = Vec::new();
     let attrs = extract_attributes_with(file_data, header, offset_size, length_size, &mut |e| {
@@ -442,23 +506,233 @@ pub fn extract_attributes_tolerant(
 
 /// Read every attribute; each one that fails goes to `on_error`, which
 /// either stops the read (returns the error) or skips that attribute.
-fn extract_attributes_with(
+fn extract_attributes_with<S: Storage + ?Sized>(
-    file_data: &[u8],
+    file_data: &S,
     header: &ObjectHeader,
     offset_size: u8,
     length_size: u8,
     on_error: &mut dyn FnMut(FormatError) -> Result<(), FormatError>,
 ) -> Result<Vec<AttributeMessage>, FormatError> {
     let mut attrs = Vec::new();
+    // Each attribute's creation order, where the file records one.
+    let mut orders: Vec<u32> = Vec::new();
 
-    // Collect compact attributes (inline in OH)
+    extract_compact_attributes(
+        file_data,
+        header,
+        offset_size,
+        length_size,
+        &mut attrs,
+        &mut orders,
+        on_error,
+    )?;
+
+    // Check for dense attributes via AttributeInfo message
+    let attr_info = find_attribute_info(header, offset_size)?;
+    if let Some(info) = &attr_info
+        && let Some(fh_addr) = info.fractal_heap_address
+    {
+        extract_dense_attributes(
+            file_data,
+            info,
+            fh_addr,
+            offset_size,
+            length_size,
+            &mut attrs,
+            &mut orders,
+            on_error,
+        )?;
+    }
+
+    // An object that tracks attribute creation order lists its attributes
+    // in that order (h5py's `track_order=True`), as libhdf5 does; otherwise
+    // they come in storage order.
+    if attr_info.is_some_and(|i| i.max_creation_index.is_some()) {
+        let mut paired: Vec<(u32, AttributeMessage)> = orders.into_iter().zip(attrs).collect();
+        paired.sort_by_key(|(o, _)| *o);
+        attrs = paired.into_iter().map(|(_, a)| a).collect();
+    }
+
+    Ok(attrs)
+}
+
+/// B-tree v2 record type of dense attribute storage's name index.
+const ATTRIBUTE_NAME_INDEX: u8 = 8;
+
+/// The attribute called `name` on the object with header `header`: the
+/// first one [`extract_attributes_tolerant`] returns under that name, or
+/// `None` if it returns none (an attribute that cannot be read is not
+/// returned there either).
+///
+/// Compact attributes are in the header and are scanned. Dense attributes
+/// are found through the name index (a v2 B-tree of lookup3 name hashes,
+/// record type 8): only the attributes whose names hash like `name` are read
+/// from the heap, O(log n) instead of all of them. Errors in the structures
+/// that index the attributes fail the call, as they fail a listing.
+pub fn find_attribute_in_file(
+    file_data: &[u8],
+    header: &ObjectHeader,
+    name: &str,
+    offset_size: u8,
+    length_size: u8,
+) -> Result<Option<AttributeMessage>, FormatError> {
+    find_attribute_core(
+        file_data,
+        header,
+        name,
+        offset_size,
+        length_size,
+        &mut Vec::new(),
+    )
+}
+
+/// [`find_attribute_in_file`] over any [`Storage`] (see
+/// [`extract_attributes_full_in`] for dense storage, whose name index still
+/// needs the whole file in memory). One with the whole file in memory is
+/// read as the slice, by code compiled in this crate (see
+/// [`crate::storage`], "Slice entry points").
+#[inline]
+pub fn find_attribute_in<S: Storage + ?Sized>(
+    file_data: &S,
+    header: &ObjectHeader,
+    name: &str,
+    offset_size: u8,
+    length_size: u8,
+) -> Result<Option<AttributeMessage>, FormatError> {
+    match file_data.as_contiguous() {
+        Some(all) => find_attribute_in_file(all, header, name, offset_size, length_size),
+        None => find_attribute_core(
+            file_data,
+            header,
+            name,
+            offset_size,
+            length_size,
+            &mut Vec::new(),
+        ),
+    }
+}
+
+/// [`find_attribute_in`], also returning the errors of the attributes it
+/// could not read on the way (which it leaves out rather than failing
+/// the call): the attribute asked for may be one of them. A reader of a
+/// file that is being written uses them to tell a read that raced the
+/// writer from an absent attribute.
+pub fn find_attribute_reporting_in<S: Storage + ?Sized>(
+    file_data: &S,
+    header: &ObjectHeader,
+    name: &str,
+    offset_size: u8,
+    length_size: u8,
+) -> Result<(Option<AttributeMessage>, Vec<FormatError>), FormatError> {
+    let mut errors = Vec::new();
+    let found = find_attribute_core(
+        file_data,
+        header,
+        name,
+        offset_size,
+        length_size,
+        &mut errors,
+    )?;
+    Ok((found, errors))
+}
+
+fn find_attribute_core<S: Storage + ?Sized>(
+    file_data: &S,
+    header: &ObjectHeader,
+    name: &str,
+    offset_size: u8,
+    length_size: u8,
+    errors: &mut Vec<FormatError>,
+) -> Result<Option<AttributeMessage>, FormatError> {
+    let attr_info = find_attribute_info(header, offset_size)?;
+    let dense = attr_info
+        .as_ref()
+        .and_then(|i| Some((i.fractal_heap_address?, i.btree_name_index_address?)));
+    let Some((fh_addr, btree_addr)) = dense else {
+        // Compact only (or dense storage without a name index, which a
+        // listing reports): as a listing finds it.
+        let (attrs, errs) =
+            extract_attributes_tolerant_in(file_data, header, offset_size, length_size)?;
+        errors.extend(errs);
+        return Ok(attrs.into_iter().find(|a| a.name == name));
+    };
+    let btree_hdr = BTreeV2Header::parse_in(
+        file_data,
+        to_usize(btree_addr)? as u64,
+        offset_size,
+        length_size,
+    )?;
+    let fh = FractalHeapHeader::parse_in(file_data, fh_addr, offset_size, length_size)?;
+    if btree_hdr.tree_type != ATTRIBUTE_NAME_INDEX || btree_hdr.record_size < 4 {
+        let (attrs, errs) =
+            extract_attributes_tolerant_in(file_data, header, offset_size, length_size)?;
+        errors.extend(errs);
+        return Ok(attrs.into_iter().find(|a| a.name == name));
+    }
+
+    // A listing has the compact attributes first.
+    let mut compact = Vec::new();
+    extract_compact_attributes(
+        file_data,
+        header,
+        offset_size,
+        length_size,
+        &mut compact,
+        &mut Vec::new(),
+        &mut |_| Ok(()),
+    )?;
+    if let Some(a) = compact.into_iter().find(|a| a.name == name) {
+        return Ok(Some(a));
+    }
+
+    // Record: heap ID + message flags(1) + creation order(4) + hash(4); the
+    // hash is the last field.
+    let hash = jenkins_lookup3(name.as_bytes());
+    let hash_at = usize::from(btree_hdr.record_size) - 4;
+    let records = find_btree_v2_records_in(file_data, &btree_hdr, offset_size, &mut |r| match r
+        .get(hash_at..hash_at + 4)
+    {
+        Some(h) => u32::from_le_bytes([h[0], h[1], h[2], h[3]]).cmp(&hash),
+        None => core::cmp::Ordering::Less,
+    })?;
+    let id_len = usize::from(fh.heap_id_length);
+    for record in &records {
+        let Some(id_bytes) = record.data.get(..id_len) else {
+            continue;
+        };
+        let attr = fh
+            .read_managed_object_in(file_data, id_bytes, offset_size)
+            .and_then(|d| {
+                AttributeMessage::parse_in_storage(&d, file_data, offset_size, length_size)
+            });
+        // One that cannot be read is left out, as from a listing.
+        match attr {
+            Ok(attr) if attr.name == name => return Ok(Some(attr)),
+            Ok(_) => {}
+            Err(e) => errors.push(e),
+        }
+    }
+    Ok(None)
+}
+
+/// The attributes stored in the object header itself (compact storage), and
+/// each one's creation order into `orders`.
+fn extract_compact_attributes<S: Storage + ?Sized>(
+    file_data: &S,
+    header: &ObjectHeader,
+    offset_size: u8,
+    length_size: u8,
+    attrs: &mut Vec<AttributeMessage>,
+    orders: &mut Vec<u32>,
+    on_error: &mut dyn FnMut(FormatError) -> Result<(), FormatError>,
+) -> Result<(), FormatError> {
     for msg in &header.messages {
         if msg.msg_type == MessageType::Attribute {
             let attr = if shared_message::is_shared(msg.flags) {
                 // Shared attribute: resolve the reference to get actual attribute data
                 shared_message::parse_shared_ref_sized(&msg.data, offset_size, length_size)
                     .and_then(|shared_ref| {
-                        shared_message::resolve_shared_message(
+                        shared_message::resolve_shared_message_in(
                             file_data,
                             &shared_ref,
                             MessageType::Attribute,
@@ -467,7 +741,7 @@ fn extract_attributes_with(
                         )
                     })
                     .and_then(|resolved| {
-                        AttributeMessage::parse_in_file(
+                        AttributeMessage::parse_in_storage(
                             &resolved,
                             file_data,
                             offset_size,
@@ -475,33 +749,19 @@ fn extract_attributes_with(
                         )
                     })
             } else {
-                AttributeMessage::parse_in_file(&msg.data, file_data, offset_size, length_size)
+                AttributeMessage::parse_in_storage(&msg.data, file_data, offset_size, length_size)
             };
             let attr = attr.and_then(|a| check_in_header(a, header));
             match attr {
-                Ok(attr) => attrs.push(attr),
+                Ok(attr) => {
+                    attrs.push(attr);
+                    orders.push(msg.creation_order.map_or(0, u32::from));
+                }
                 Err(e) => on_error(e)?,
             }
         }
     }
-
+    Ok(())
-    // Check for dense attributes via AttributeInfo message
-    let attr_info = find_attribute_info(header, offset_size)?;
-    if let Some(info) = attr_info
-        && let Some(fh_addr) = info.fractal_heap_address
-    {
-        extract_dense_attributes(
-            file_data,
-            &info,
-            fh_addr,
-            offset_size,
-            length_size,
-            &mut attrs,
-            on_error,
-        )?;
-    }
-
-    Ok(attrs)
 }
 
 /// Find and parse the Attribute Info message from an object header.
@@ -518,18 +778,21 @@ fn find_attribute_info(
     Ok(None)
 }
 
-/// Extract attributes from dense storage (fractal heap + B-tree v2).
+/// Extract attributes from dense storage (fractal heap + B-tree v2), and
-fn extract_dense_attributes(
+/// each one's creation order into `orders`.
-    file_data: &[u8],
+#[allow(clippy::too_many_arguments)]
+fn extract_dense_attributes<S: Storage + ?Sized>(
+    file_data: &S,
     attr_info: &AttributeInfoMessage,
     fh_addr: u64,
     offset_size: u8,
     length_size: u8,
     attrs: &mut Vec<AttributeMessage>,
+    orders: &mut Vec<u32>,
     on_error: &mut dyn FnMut(FormatError) -> Result<(), FormatError>,
 ) -> Result<(), FormatError> {
     // Parse fractal heap
-    let fh = FractalHeapHeader::parse(file_data, fh_addr as usize, offset_size, length_size)?;
+    let fh = FractalHeapHeader::parse_in(file_data, fh_addr, offset_size, length_size)?;
 
     // Parse B-tree v2 for name index (type 8)
     let btree_addr = attr_info
@@ -538,8 +801,13 @@ fn extract_dense_attributes(
             expected: 1,
             available: 0,
         })?;
-    let btree_hdr = BTreeV2Header::parse(file_data, btree_addr as usize, offset_size, length_size)?;
+    let btree_hdr = BTreeV2Header::parse_in(
-    let records = collect_btree_v2_records(file_data, &btree_hdr, offset_size, length_size)?;
+        file_data,
+        to_usize(btree_addr)? as u64,
+        offset_size,
+        length_size,
+    )?;
+    let records = collect_btree_v2_records_in(file_data, &btree_hdr, offset_size, length_size)?;
 
     for record in &records {
         // Per HDF5 spec, both type 8 and type 9 records start with heap_id:
@@ -556,12 +824,19 @@ fn extract_dense_attributes(
 
         // The data in the heap is a complete attribute message
         let attr = fh
-            .read_managed_object(file_data, id_bytes, offset_size)
+            .read_managed_object_in(file_data, id_bytes, offset_size)
             .and_then(|attr_data| {
-                AttributeMessage::parse_in_file(&attr_data, file_data, offset_size, length_size)
+                AttributeMessage::parse_in_storage(&attr_data, file_data, offset_size, length_size)
             });
         match attr {
-            Ok(attr) => attrs.push(attr),
+            Ok(attr) => {
+                attrs.push(attr);
+                let order = record
+                    .data
+                    .get(id_len + 1..id_len + 5)
+                    .map_or(0, |b| u32::from_le_bytes([b[0], b[1], b[2], b[3]]));
+                orders.push(order);
+            }
             Err(e) => on_error(e)?,
         }
     }
@@ -961,4 +1236,73 @@ mod tests {
         let strs = attr.read_as_strings().unwrap();
         assert_eq!(strs, vec!["abcd", "EFGH"]);
     }
+
+    /// Every object's attributes in h5py-written files read identically
+    /// through a read_at-only CountingStorage — compact ones, shared ones,
+    /// those behind an Attribute Info message and dense storage (its v2
+    /// B-tree name index included) — and through a slice as Storage.
+    #[test]
+    fn storage_reads_match_slice_reads() {
+        use crate::storage::CountingStorage;
+        let files: [(&str, &[u8]); 5] = [
+            ("attrs", include_bytes!("../tests/fixtures/attrs.h5")),
+            (
+                "mixed_attrs",
+                include_bytes!("../tests/fixtures/mixed_attrs.h5"),
+            ),
+            (
+                "dense_attrs",
+                include_bytes!("../tests/fixtures/dense_attrs.h5"),
+            ),
+            (
+                "dense_attrs_root",
+                include_bytes!("../tests/fixtures/dense_attrs_root.h5"),
+            ),
+            (
+                "shared_fill_value",
+                include_bytes!("../tests/fixtures/shared_fill_value.h5"),
+            ),
+        ];
+        let (mut same, mut dense, mut attrs) = (0, 0, 0);
+        for (name, file) in files {
+            let sb = crate::superblock::Superblock::parse(file, 0).unwrap();
+            let (os, ls) = (sb.offset_size, sb.length_size);
+            let mut addrs = vec![sb.root_group_address];
+            addrs.extend(
+                crate::group_v2::resolve_group_children(file, &sb, sb.root_group_address)
+                    .unwrap()
+                    .iter()
+                    .map(|e| e.object_header_address),
+            );
+            let storage = CountingStorage::new(file.to_vec());
+            for addr in addrs {
+                let header = ObjectHeader::parse(file, addr as usize, os, ls).unwrap();
+                let want = extract_attributes_full(file, &header, os, ls);
+                let slice_storage = extract_attributes_full_in(&file, &header, os, ls);
+                assert_eq!(format!("{slice_storage:?}"), format!("{want:?}"));
+                let got = extract_attributes_full_in(&storage, &header, os, ls);
+                let got_t = extract_attributes_tolerant_in(&storage, &header, os, ls);
+                let is_dense = find_attribute_info(&header, os)
+                    .unwrap()
+                    .is_some_and(|i| i.fractal_heap_address.is_some());
+                if is_dense {
+                    dense += 1;
+                }
+                attrs += want.as_ref().map_or(0, Vec::len);
+                assert_eq!(format!("{got:?}"), format!("{want:?}"), "{name}");
+                let want_t = extract_attributes_tolerant(file, &header, os, ls);
+                assert_eq!(format!("{got_t:?}"), format!("{want_t:?}"), "{name}");
+                same += 1;
+                for a in want.iter().flatten() {
+                    let one = find_attribute_in(&storage, &header, &a.name, os, ls);
+                    let want_one = find_attribute_in_file(file, &header, &a.name, os, ls);
+                    assert_eq!(format!("{one:?}"), format!("{want_one:?}"), "{name}");
+                }
+            }
+        }
+        assert!(
+            same >= 5 && dense >= 2 && attrs >= 5,
+            "{same} {dense} {attrs}"
+        );
+    }
 }

crates/clawhdf5-format/src/btree_v1.rs

@@ -4,6 +4,7 @@
 use alloc::vec::Vec;
 
 use crate::error::FormatError;
+use crate::storage::{Storage, read_exact_at};
 
 /// A parsed B-tree v1 node.
 #[derive(Debug, Clone)]
@@ -74,13 +75,30 @@ impl BTreeV1Node {
         file_data: &[u8],
         offset: usize,
         offset_size: u8,
+        length_size: u8,
+    ) -> Result<BTreeV1Node, FormatError> {
+        Self::parse_in(file_data, offset as u64, offset_size, length_size)
+    }
+
+    /// [`Self::parse`] over any [`Storage`]: one read of the node's header,
+    /// one of its keys and children.
+    pub fn parse_in<S: Storage + ?Sized>(
+        file: &S,
+        offset: u64,
+        offset_size: u8,
         _length_size: u8,
     ) -> Result<BTreeV1Node, FormatError> {
         // signature(4) + node_type(1) + node_level(1) + entries_used(2) = 8
         // + left_sibling(offset_size) + right_sibling(offset_size)
         let os = offset_size as usize;
         let header_size = 8 + os * 2;
-        ensure_len(file_data, offset, header_size)?;
+        // The body is read once the header says how long it is.
+        file.hint(offset, NODE_HINT_LEN);
+        let header = read_exact_at(file, offset, header_size)?;
+        let file_data: &[u8] = &header;
+        // The header's read checked that `offset + header_size` fits.
+        let body_start = offset + header_size as u64;
+        let offset = 0usize;
 
         if &file_data[offset..offset + 4] != b"TREE" {
             return Err(FormatError::InvalidBTreeSignature);
@@ -102,31 +120,30 @@ impl BTreeV1Node {
         } else {
             Some(read_offset(file_data, pos, offset_size)?)
         };
-        pos += os;
 
         // For type 0: keys are offset_size bytes, children are offset_size bytes
         // Layout: key[0], child[0], key[1], child[1], ..., key[N-1], child[N-1], key[N]
         let eu = entries_used as usize;
         let key_size = os; // For type 0, key = offset_size
         let needed = eu * (key_size + os) + key_size; // eu children + (eu+1) keys
-        ensure_len(file_data, pos, needed)?;
+        let body = read_exact_at(file, body_start, needed)?;
+        let file_data: &[u8] = &body;
 
         let mut keys = Vec::with_capacity(eu + 1);
         let mut children = Vec::with_capacity(eu);
 
-        for _i in 0..eu {
+        if os == 0 {
-            // key[i]
+            // What reading the first key reports (and keeps `chunks_exact`
-            let key = read_offset(file_data, pos, offset_size)?;
+            // below from being given a zero size).
-            keys.push(key);
+            return Err(FormatError::InvalidOffsetSize(offset_size));
-            pos += key_size;
-            // child[i]
-            let child = read_offset(file_data, pos, offset_size)?;
-            children.push(child);
-            pos += os;
         }
-        // final key
+        // `needed` bytes: key[0], child[0], ..., child[eu - 1], key[eu].
-        let key = read_offset(file_data, pos, offset_size)?;
+        let (pairs, last) = file_data.split_at(eu * (key_size + os));
-        keys.push(key);
+        for pair in pairs.chunks_exact(key_size + os) {
+            keys.push(read_offset(pair, 0, offset_size)?);
+            children.push(read_offset(pair, key_size, offset_size)?);
+        }
+        keys.push(read_offset(last, 0, offset_size)?);
 
         Ok(BTreeV1Node {
             node_type,
@@ -141,7 +158,18 @@ impl BTreeV1Node {
 }
 
 /// Maximum recursion depth for B-tree traversal (malformed data protection).
-const MAX_BTREE_DEPTH: usize = 64;
+pub(crate) const MAX_BTREE_DEPTH: usize = 64;
+
+/// What a symbol table node takes with libhdf5's default group leaf K (4):
+/// its 8-byte header and 2K entries of 40 bytes (8-byte offsets). Hinted
+/// before one is read ([`Storage::hint`]); a node of another size is read
+/// all the same.
+const SNOD_HINT_LEN: usize = 8 + 8 * 40;
+
+/// What a group B-tree node takes with libhdf5's default internal K (16):
+/// its header (24 bytes with 8-byte offsets), 2K + 1 keys and 2K children
+/// of 8 bytes. Hinted before one is read.
+const NODE_HINT_LEN: usize = 24 + (2 * 16 + 1 + 2 * 16) * 8;
 
 /// Collect all leaf-level child addresses (SNOD addresses) by traversing the B-tree.
 pub fn collect_symbol_table_nodes(
@@ -150,11 +178,21 @@ pub fn collect_symbol_table_nodes(
     offset_size: u8,
     length_size: u8,
 ) -> Result<Vec<u64>, FormatError> {
-    collect_symbol_table_nodes_inner(file_data, btree_address, offset_size, length_size, 0)
+    collect_symbol_table_nodes_in(file_data, btree_address, offset_size, length_size)
 }
 
-fn collect_symbol_table_nodes_inner(
+/// [`collect_symbol_table_nodes`] over any [`Storage`]: two reads per node.
-    file_data: &[u8],
+pub fn collect_symbol_table_nodes_in<S: Storage + ?Sized>(
+    file: &S,
+    btree_address: u64,
+    offset_size: u8,
+    length_size: u8,
+) -> Result<Vec<u64>, FormatError> {
+    collect_symbol_table_nodes_inner(file, btree_address, offset_size, length_size, 0)
+}
+
+fn collect_symbol_table_nodes_inner<S: Storage + ?Sized>(
+    file: &S,
     btree_address: u64,
     offset_size: u8,
     length_size: u8,
@@ -164,29 +202,47 @@ fn collect_symbol_table_nodes_inner(
         return Err(FormatError::NestingDepthExceeded);
     }
 
-    let node = BTreeV1Node::parse(file_data, btree_address as usize, offset_size, length_size)?;
+    let node = BTreeV1Node::parse_in(file, btree_address, offset_size, length_size)?;
 
     if node.node_type != 0 {
         return Err(FormatError::InvalidBTreeNodeType(node.node_type));
     }
 
     if node.node_level == 0 {
-        // Leaf: children are SNOD addresses
+        // Leaf: children are SNOD addresses, read next (see
+        // `Storage::hint`).
+        for &snod in &node.children {
+            file.hint(snod, SNOD_HINT_LEN);
+        }
         Ok(node.children)
     } else {
-        // Internal: recurse into children
+        // Internal: recurse into children. A child that fails does not
+        // stop the walk: the others are still descended into (reading, not
+        // using, what they hold), then the first error is returned. The
+        // result and the error are those of stopping at the first failure;
+        // a storage that records what it lacks (see `storage::touch`)
+        // learns every node the walk can reach in one attempt.
         let mut result = Vec::new();
+        let mut failed = None;
         for &child_addr in &node.children {
-            let child_snods = collect_symbol_table_nodes_inner(
+            match collect_symbol_table_nodes_inner(
-                file_data,
+                file,
                 child_addr,
                 offset_size,
                 length_size,
                 depth + 1,
-            )?;
+            ) {
-            result.extend(child_snods);
+                Ok(child_snods) if failed.is_none() => result.extend(child_snods),
+                Ok(_) => {}
+                Err(e) => {
+                    failed.get_or_insert(e);
+                }
+            }
+        }
+        match failed {
+            Some(e) => Err(e),
+            None => Ok(result),
         }
-        Ok(result)
     }
 }
 
@@ -317,4 +373,48 @@ mod tests {
         assert_eq!(node.entries_used, 1);
         assert_eq!(node.children, vec![0x50]);
     }
+
+    /// Nodes and trees, cut at every length, parse identically through a
+    /// `read_at`-only storage.
+    #[test]
+    fn storage_parse_matches_slice_parse() {
+        use crate::storage::CountingStorage;
+        let nodes = [
+            build_btree_node(0, 0, &[0, 5, 10], &[0x100, 0x200], None, None, 8),
+            build_btree_node(0, 0, &[0, 5], &[0x100], Some(0x40), Some(0x80), 4),
+            build_btree_node(1, 2, &[0, 5], &[0x100], None, Some(0x80), 8),
+        ];
+        for (n, node) in nodes.iter().enumerate() {
+            let os = if n == 1 { 4 } else { 8 };
+            for cut in 0..=node.len() {
+                let f = &node[..cut];
+                let storage = CountingStorage::new(f.to_vec());
+                let want = BTreeV1Node::parse(f, 0, os, 8);
+                let got = BTreeV1Node::parse_in(&storage, 0, os, 8);
+                assert_eq!(format!("{got:?}"), format!("{want:?}"));
+            }
+        }
+        let leaf1 = build_btree_node(0, 0, &[0, 5], &[0xA00], None, None, 8);
+        let leaf2 = build_btree_node(0, 0, &[5, 10], &[0xB00], None, None, 8);
+        let internal = build_btree_node(0, 1, &[0, 5, 10], &[0, 256], None, None, 8);
+        let mut file = vec![0u8; 512 + internal.len()];
+        file[..leaf1.len()].copy_from_slice(&leaf1);
+        file[256..256 + leaf2.len()].copy_from_slice(&leaf2);
+        file[512..].copy_from_slice(&internal);
+        for cut in [file.len(), 300, 260, 100, 10] {
+            let mut f = file.clone();
+            if cut < 512 {
+                // Truncate the leaves, keep the root.
+                f[cut..512].fill(0);
+            }
+            let storage = CountingStorage::new(f.clone());
+            assert_eq!(
+                collect_symbol_table_nodes_in(&storage, 512, 8, 8),
+                collect_symbol_table_nodes(&f, 512, 8, 8)
+            );
+        }
+        let storage = CountingStorage::new(file);
+        collect_symbol_table_nodes_in(&storage, 512, 8, 8).unwrap();
+        assert_eq!(storage.reads(), 6);
+    }
 }

crates/clawhdf5-format/src/btree_v1_write.rs

@@ -0,0 +1,470 @@
+//! Writing a version-1 B-tree chunk index (node type 1): the chunk index of
+//! layout message versions 1-3, and the only one HDF5 1.8 reads.
+//!
+//! The tree is built the way libhdf5 builds it when the chunks reach it one
+//! after another in row-major order (a whole-dataset `H5Dwrite` of a 1-D
+//! dataset, or of any dataset without a chunk cache; with one, libhdf5
+//! inserts the small chunks of a multi-dimensional dataset in the order its
+//! cache evicts them, which fills the nodes differently): each
+//! chunk goes through the same steps as `H5B_insert` (`H5B.c`) with the
+//! chunk callbacks of `H5Dbtree.c`, so nodes split where libhdf5's split,
+//! with its default split ratios (a full right-most node keeps 90% of its
+//! children, a left-most one 10%, any other half), and keys hold what
+//! libhdf5's hold:
+//!
+//! - a chunk's key is its size in the file, its filter mask and its offsets
+//!   (the element-size coordinate 0);
+//! - a node's final key is the zero-size key one chunk past the chunk that
+//!   last moved it (every scaled coordinate plus one, `H5D__btree_new_node`),
+//!   which libhdf5 moves only when a new chunk is not below it
+//!   (`H5D__btree_cmp3`) — so after an even number of appends in one
+//!   dimension it lies on the last chunk itself;
+//! - a full root is copied to a new node and becomes the parent of the copy
+//!   and its new sibling, so the root's address (the layout message's) never
+//!   changes.
+//!
+//! Nodes are laid out in the order libhdf5 allocates them (the root first,
+//! then each new node as a split creates it), all of the full node size, the
+//! unused slots zero.
+
+#[cfg(not(feature = "std"))]
+use alloc::{format, vec, vec::Vec};
+
+use core::cmp::Ordering;
+
+use crate::error::FormatError;
+
+/// libhdf5's default chunk B-tree K (`HDF5_BTREE_CHUNK_IK_DEF`): nodes hold
+/// up to 2K = 64 children. Superblocks of version 2 cannot record another
+/// value without a superblock extension, which this writer does not emit.
+pub(crate) const CHUNK_BTREE_K: u16 = 32;
+
+/// libhdf5's default split ratios (`H5D_XFER_BTREE_SPLIT_RATIO_DEF`) for a
+/// left-most, middle and right-most node.
+const SPLIT_RATIOS: [f64; 3] = [0.1, 0.5, 0.9];
+
+/// A chunk to index: scaled coordinates (offset / chunk dimension) in each
+/// dataset dimension, stored size, filter mask and address.
+pub(crate) struct ChunkEntry {
+    pub(crate) scaled: Vec<u64>,
+    pub(crate) nbytes: u64,
+    pub(crate) filter_mask: u32,
+    pub(crate) address: u64,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+struct Key {
+    nbytes: u32,
+    mask: u32,
+    /// Scaled coordinates, the element-size one (0 or 1) last.
+    scaled: Vec<u64>,
+}
+
+impl Key {
+    /// `H5D__btree_new_node`'s right key: one chunk past `self` in every
+    /// dimension, with no storage.
+    fn right_of(&self) -> Key {
+        Key {
+            nbytes: 0,
+            mask: 0,
+            scaled: self.scaled.iter().map(|s| s + 1).collect(),
+        }
+    }
+
+    fn cmp_scaled(&self, other: &Key) -> Ordering {
+        self.scaled.cmp(&other.scaled)
+    }
+}
+
+#[derive(Debug, Clone)]
+struct Node {
+    level: u8,
+    left: Option<usize>,
+    right: Option<usize>,
+    /// `children.len() + 1` keys once the node holds a child.
+    keys: Vec<Key>,
+    /// Chunk addresses in a leaf, node indexes above.
+    children: Vec<u64>,
+}
+
+/// What an insertion below a node did (`H5B__insert_helper`'s outputs).
+#[derive(Default)]
+struct Ret {
+    /// The node's new left key (`lt_key_changed`).
+    lt: Option<Key>,
+    /// The node's new right key (`rt_key_changed`).
+    rt: Option<Key>,
+    /// The node split: the key shared by the halves and the new right node.
+    split: Option<(Key, usize)>,
+}
+
+struct Tree {
+    nodes: Vec<Node>,
+    two_k: usize,
+}
+
+fn bad(why: &str) -> FormatError {
+    FormatError::SerializationError(format!("version-1 B-tree chunk index: {why}"))
+}
+
+impl Tree {
+    fn new(k: u16) -> Self {
+        Self {
+            nodes: vec![Node {
+                level: 0,
+                left: None,
+                right: None,
+                keys: Vec::new(),
+                children: Vec::new(),
+            }],
+            two_k: 2 * usize::from(k),
+        }
+    }
+
+    /// `H5B_insert` of `key` (a chunk after every chunk already inserted).
+    fn insert(&mut self, key: &Key, addr: u64) -> Result<(), FormatError> {
+        let r = self.insert_helper(0, key, addr, 64)?;
+        let Some((md, split)) = r.split else {
+            return Ok(());
+        };
+        // The root split: copy it to a new node and make the root the
+        // parent of the copy and its new right sibling.
+        let lt = r.lt.unwrap_or_else(|| self.nodes[0].keys[0].clone());
+        let rt = match r.rt {
+            Some(rt) => rt,
+            None => self.nodes[split]
+                .keys
+                .last()
+                .cloned()
+                .ok_or_else(|| bad("empty node"))?,
+        };
+        let moved = self.nodes[0].clone();
+        let level = moved.level;
+        let moved_id = self.nodes.len();
+        self.nodes.push(moved);
+        self.nodes[split].left = Some(moved_id);
+        self.nodes[0] = Node {
+            level: level + 1,
+            left: None,
+            right: None,
+            keys: vec![lt, md, rt],
+            children: vec![moved_id as u64, split as u64],
+        };
+        Ok(())
+    }
+
+    fn insert_helper(
+        &mut self,
+        id: usize,
+        key: &Key,
+        addr: u64,
+        depth: u8,
+    ) -> Result<Ret, FormatError> {
+        if depth == 0 {
+            return Err(bad("tree too deep"));
+        }
+        let n = self.nodes[id].children.len();
+        let level = self.nodes[id].level;
+        let mut ret = Ret::default();
+        if n == 0 {
+            // The first chunk (H5B_INS_FIRST): its key and the right key.
+            let node = &mut self.nodes[id];
+            node.keys = vec![key.clone(), key.right_of()];
+            node.children = vec![addr];
+            return Ok(ret);
+        }
+        // Binary search with H5D__btree_cmp3: 1 when the chunk is not below
+        // the right key, -1 when below the left key, else 0.
+        let (mut lo, mut hi, mut idx) = (0usize, n, 0usize);
+        let mut cmp = Ordering::Less;
+        while lo < hi && cmp != Ordering::Equal {
+            idx = (lo + hi) / 2;
+            let node = &self.nodes[id];
+            cmp = if key.cmp_scaled(&node.keys[idx + 1]) != Ordering::Less {
+                Ordering::Greater
+            } else if key.cmp_scaled(&node.keys[idx]) == Ordering::Less {
+                Ordering::Less
+            } else {
+                Ordering::Equal
+            };
+            if cmp == Ordering::Less {
+                hi = idx;
+            } else {
+                lo = idx + 1;
+            }
+        }
+        let (mut lt_changed, mut rt_changed) = (false, false);
+        // The child to add after child `idx`, with its left key.
+        let mut new_child: Option<(Key, u64)> = None;
+        match cmp {
+            Ordering::Less => return Err(bad("chunks out of order")),
+            Ordering::Greater if idx + 1 < n => {
+                return Err(bad("cannot place chunk"));
+            }
+            Ordering::Greater if level == 0 => {
+                // Past every chunk of the right-most leaf: a new maximum
+                // (H5B_INS_RIGHT through `new_node`), which moves the right
+                // key one chunk past it.
+                idx = n - 1;
+                self.nodes[id].keys[idx + 1] = key.right_of();
+                rt_changed = true;
+                new_child = Some((key.clone(), addr));
+            }
+            Ordering::Equal if level == 0 => {
+                // Inside the last chunk's range: H5D__btree_insert adds it
+                // to the right of that chunk; the right key stays.
+                if key.scaled == self.nodes[id].keys[idx].scaled {
+                    return Err(bad("duplicate chunk"));
+                }
+                new_child = Some((key.clone(), addr));
+            }
+            _ => {
+                if cmp == Ordering::Greater {
+                    idx = n - 1;
+                }
+                let child = usize::try_from(self.nodes[id].children[idx])
+                    .map_err(|_| bad("bad node index"))?;
+                let r = self.insert_helper(child, key, addr, depth - 1)?;
+                if let Some(lt) = r.lt {
+                    self.nodes[id].keys[idx] = lt;
+                    lt_changed = true;
+                }
+                if let Some(rt) = r.rt {
+                    self.nodes[id].keys[idx + 1] = rt;
+                    rt_changed = true;
+                }
+                if let Some((md, split)) = r.split {
+                    new_child = Some((md, split as u64));
+                }
+            }
+        }
+        // Pass the node's changed end keys up, as H5B__insert_helper does.
+        if lt_changed && idx == 0 {
+            ret.lt = Some(self.nodes[id].keys[0].clone());
+        }
+        if rt_changed && idx + 1 >= n {
+            ret.rt = Some(self.nodes[id].keys[idx + 1].clone());
+        }
+        if let Some((md, child)) = new_child {
+            // A full node splits first; the child goes to the half that
+            // holds child `idx`.
+            let (mut target, mut split) = (id, None);
+            if n == self.two_k {
+                let s = self.split(id, idx);
+                let nleft = self.nodes[id].children.len();
+                if idx >= nleft {
+                    idx -= nleft;
+                    target = s;
+                }
+                split = Some(s);
+            }
+            // H5B__insert_child (H5B_INS_RIGHT): the new child after child
+            // `idx`, its left key after that child's.
+            let node = &mut self.nodes[target];
+            node.keys.insert(idx + 1, md);
+            node.children.insert(idx + 1, child);
+            ret.split = split.map(|s| (self.nodes[s].keys[0].clone(), s));
+        }
+        Ok(ret)
+    }
+
+    /// `H5B__split` of the full node `id`, the insertion going after child
+    /// `idx`; returns the new right node.
+    fn split(&mut self, id: usize, idx: usize) -> usize {
+        let node = &self.nodes[id];
+        let ratio = if node.right.is_none() {
+            SPLIT_RATIOS[2]
+        } else if node.left.is_none() {
+            SPLIT_RATIOS[0]
+        } else {
+            SPLIT_RATIOS[1]
+        };
+        let mut nleft = (self.two_k as f64 * ratio) as usize;
+        if idx < nleft && nleft == self.two_k {
+            nleft -= 1;
+        } else if idx >= nleft && nleft == 0 {
+            nleft += 1;
+        }
+        let new_id = self.nodes.len();
+        let right = Node {
+            level: node.level,
+            left: Some(id),
+            right: node.right,
+            keys: node.keys[nleft..].to_vec(),
+            children: node.children[nleft..].to_vec(),
+        };
+        let old_right = node.right;
+        self.nodes.push(right);
+        if let Some(r) = old_right {
+            self.nodes[r].left = Some(new_id);
+        }
+        let node = &mut self.nodes[id];
+        node.keys.truncate(nleft + 1);
+        node.children.truncate(nleft);
+        node.right = Some(new_id);
+        new_id
+    }
+}
+
+/// Bytes of one node of a chunk B-tree with `ndims` key dimensions (the
+/// dataset's rank plus the element-size one).
+fn node_size(two_k: usize, ndims: usize, offset_size: usize) -> usize {
+    let key = 8 + 8 * ndims;
+    8 + 2 * offset_size + (two_k + 1) * key + two_k * offset_size
+}
+
+/// Build the chunk B-tree for `chunks`, given in row-major order of their
+/// scaled coordinates, with nodes laid out from `base_address`. `chunk_dims`
+/// are the chunk's dimensions (the dataset's rank of them) and `elem_size`
+/// the element size, the key's last dimension. Returns the nodes' bytes; the
+/// root is at `base_address`. `chunks` must not be empty: an index without
+/// chunks has no tree (its address is undefined).
+pub(crate) fn build_chunk_btree_v1_at(
+    chunks: &[ChunkEntry],
+    chunk_dims: &[u64],
+    elem_size: u32,
+    base_address: u64,
+    offset_size: u8,
+) -> Result<Vec<u8>, FormatError> {
+    if chunks.is_empty() {
+        return Err(bad("no chunks"));
+    }
+    let rank = chunk_dims.len();
+    let mut tree = Tree::new(CHUNK_BTREE_K);
+    for c in chunks {
+        if c.scaled.len() != rank {
+            return Err(bad("chunk rank differs from the dataset's"));
+        }
+        let nbytes = u32::try_from(c.nbytes).map_err(|_| {
+            FormatError::SerializationError(format!(
+                "a chunk of {} bytes cannot be indexed by a version-1 B-tree \
+                 (HDF5 1.8 chunks are under 4 GiB)",
+                c.nbytes
+            ))
+        })?;
+        let mut scaled = c.scaled.clone();
+        scaled.push(0);
+        let key = Key {
+            nbytes,
+            mask: c.filter_mask,
+            scaled,
+        };
+        tree.insert(&key, c.address)?;
+    }
+
+    let os = usize::from(offset_size);
+    let ndims = rank + 1;
+    let nsize = node_size(tree.two_k, ndims, os);
+    let addr_of = |id: usize| base_address + (id * nsize) as u64;
+    let mut dims: Vec<u64> = chunk_dims.to_vec();
+    dims.push(u64::from(elem_size));
+    let mut out = vec![0u8; tree.nodes.len() * nsize];
+    for (i, node) in tree.nodes.iter().enumerate() {
+        let d = &mut out[i * nsize..(i + 1) * nsize];
+        d[0..4].copy_from_slice(b"TREE");
+        d[4] = 1; // node type: raw data chunks
+        d[5] = node.level;
+        let n = u16::try_from(node.children.len()).map_err(|_| bad("node too large"))?;
+        d[6..8].copy_from_slice(&n.to_le_bytes());
+        let undef = u64::MAX;
+        put_addr(&mut d[8..], node.left.map_or(undef, addr_of), os);
+        put_addr(&mut d[8 + os..], node.right.map_or(undef, addr_of), os);
+        let mut p = 8 + 2 * os;
+        for (k, key) in node.keys.iter().enumerate() {
+            d[p..p + 4].copy_from_slice(&key.nbytes.to_le_bytes());
+            d[p + 4..p + 8].copy_from_slice(&key.mask.to_le_bytes());
+            for (j, (&s, &dim)) in key.scaled.iter().zip(&dims).enumerate() {
+                let off = s
+                    .checked_mul(dim)
+                    .ok_or_else(|| FormatError::Overflow("chunk key offset".into()))?;
+                d[p + 8 + 8 * j..p + 16 + 8 * j].copy_from_slice(&off.to_le_bytes());
+            }
+            p += 8 + 8 * ndims;
+            if let Some(&child) = node.children.get(k) {
+                let a = if node.level == 0 {
+                    child
+                } else {
+                    addr_of(usize::try_from(child).map_err(|_| bad("bad node index"))?)
+                };
+                put_addr(&mut d[p..], a, os);
+                p += os;
+            }
+        }
+    }
+    Ok(out)
+}
+
+fn put_addr(d: &mut [u8], v: u64, os: usize) {
+    d[..os].copy_from_slice(&v.to_le_bytes()[..os]);
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn build(n: u64) -> Tree {
+        let mut t = Tree::new(CHUNK_BTREE_K);
+        for i in 0..n {
+            let key = Key {
+                nbytes: 80,
+                mask: 0,
+                scaled: vec![i, 0],
+            };
+            t.insert(&key, 1000 + i).unwrap();
+        }
+        t
+    }
+
+    /// Leaves in order from the root, with their child counts.
+    fn leaves(t: &Tree, id: usize, out: &mut Vec<usize>) {
+        let n = &t.nodes[id];
+        if n.level == 0 {
+            out.push(n.children.len());
+        } else {
+            for &c in &n.children {
+                leaves(t, c as usize, out);
+            }
+        }
+    }
+
+    #[test]
+    fn sequential_appends_split_as_libhdf5_does() {
+        // libhdf5 2.0 (h5py, libver=('v108', 'latest')) writes 1000 chunks
+        // as a root over 17 leaves of 57 chunks and one of 31, with the
+        // root's right key on the last chunk (9990, 8 for 10-element f8
+        // chunks).
+        let t = build(1000);
+        assert_eq!(t.nodes[0].level, 1);
+        let mut l = Vec::new();
+        leaves(&t, 0, &mut l);
+        let mut want = vec![57; 17];
+        want.push(31);
+        assert_eq!(l, want);
+        assert_eq!(t.nodes[0].keys.last().unwrap().scaled, vec![999, 1]);
+        // 100 000 chunks: three levels, a root of 31 children.
+        let t = build(100_000);
+        assert_eq!(t.nodes[0].level, 2);
+        assert_eq!(t.nodes[0].children.len(), 31);
+    }
+
+    #[test]
+    fn right_key_moves_every_other_append() {
+        let t = build(5);
+        assert_eq!(t.nodes[0].keys.last().unwrap().scaled, vec![5, 1]);
+        let t = build(6);
+        assert_eq!(t.nodes[0].keys.last().unwrap().scaled, vec![5, 1]);
+    }
+
+    #[test]
+    fn keys_and_siblings_are_consistent() {
+        let t = build(5000);
+        for (i, n) in t.nodes.iter().enumerate() {
+            assert!(n.children.len() <= t.two_k);
+            assert_eq!(n.keys.len(), n.children.len() + 1);
+            if let Some(r) = n.right {
+                assert_eq!(t.nodes[r].left, Some(i));
+                assert_eq!(n.keys.last(), t.nodes[r].keys.first());
+            }
+        }
+    }
+}

crates/clawhdf5-format/src/btree_v2.rs

@@ -2,11 +2,14 @@
 
 #[cfg(not(feature = "std"))]
 use alloc::vec::Vec;
+use core::cmp::Ordering;
 
 #[cfg(feature = "checksum")]
 use byteorder::{ByteOrder, LittleEndian};
 
+use crate::addr::to_usize;
 use crate::error::FormatError;
+use crate::storage::{Storage, Window, len_usize};
 
 /// Parsed B-tree v2 header (signature "BTHD").
 #[derive(Debug, Clone)]
@@ -71,7 +74,7 @@ fn ensure_len(data: &[u8], pos: usize, needed: usize) -> Result<(), FormatError>
 
 /// Compute the number of bytes needed to represent a count, using variable-width encoding.
 /// B-tree v2 uses this for the number of records fields in internal nodes.
-fn bytes_for_max_records(max_nrec: u64) -> usize {
+pub(crate) fn bytes_for_max_records(max_nrec: u64) -> usize {
     if max_nrec == 0 {
         return 1;
     }
@@ -97,38 +100,52 @@ impl BTreeV2Header {
         offset_size: u8,
         length_size: u8,
     ) -> Result<BTreeV2Header, FormatError> {
-        ensure_len(file_data, offset, 4)?;
+        Self::parse_in(file_data, offset as u64, offset_size, length_size)
-        if &file_data[offset..offset + 4] != b"BTHD" {
+    }
+
+    /// [`Self::parse`] over any [`Storage`]: one bounded read of the
+    /// header.
+    pub fn parse_in<S: Storage + ?Sized>(
+        file: &S,
+        offset: u64,
+        offset_size: u8,
+        length_size: u8,
+    ) -> Result<BTreeV2Header, FormatError> {
+        // Every field and the checksum; the window holds all of it or ends
+        // at the end of the file, so its bounds checks are the whole-file
+        // ones.
+        let full = 16 + usize::from(offset_size) + 2 + usize::from(length_size) + 4;
+        let w = Window::read(file, offset, full)?;
+        let d = &w.bytes;
+        w.ensure(0, 4)?;
+        if &d[..4] != b"BTHD" {
             return Err(FormatError::InvalidBTreeV2Signature);
         }
 
-        ensure_len(file_data, offset, 4 + 1 + 1 + 4 + 2 + 2 + 1 + 1)?;
+        w.ensure(0, 4 + 1 + 1 + 4 + 2 + 2 + 1 + 1)?;
-        let version = file_data[offset + 4];
+        let version = d[4];
         if version != 0 {
             return Err(FormatError::InvalidBTreeV2Version(version));
         }
 
-        let tree_type = file_data[offset + 5];
+        let tree_type = d[5];
-        let node_size = u32::from_le_bytes([
+        let node_size = u32::from_le_bytes([d[6], d[7], d[8], d[9]]);
-            file_data[offset + 6],
+        let record_size = u16::from_le_bytes([d[10], d[11]]);
-            file_data[offset + 7],
+        let depth = u16::from_le_bytes([d[12], d[13]]);
-            file_data[offset + 8],
+        let _split_percent = d[14];
-            file_data[offset + 9],
+        let _merge_percent = d[15];
-        ]);
-        let record_size = u16::from_le_bytes([file_data[offset + 10], file_data[offset + 11]]);
-        let depth = u16::from_le_bytes([file_data[offset + 12], file_data[offset + 13]]);
-        let _split_percent = file_data[offset + 14];
-        let _merge_percent = file_data[offset + 15];
 
-        let mut pos = offset + 16;
+        let mut pos = 16;
-        let root_node_address = read_offset(file_data, pos, offset_size)?;
+        w.ensure(pos, usize::from(offset_size))?;
+        let root_node_address = read_offset(d, pos, offset_size)?;
         pos += offset_size as usize;
 
-        ensure_len(file_data, pos, 2)?;
+        w.ensure(pos, 2)?;
-        let num_records_in_root = u16::from_le_bytes([file_data[pos], file_data[pos + 1]]);
+        let num_records_in_root = u16::from_le_bytes([d[pos], d[pos + 1]]);
         pos += 2;
 
-        let total_records = read_offset(file_data, pos, length_size)?;
+        w.ensure(pos, usize::from(length_size))?;
+        let total_records = read_offset(d, pos, length_size)?;
         #[allow(unused_assignments)]
         {
             pos += length_size as usize;
@@ -137,9 +154,9 @@ impl BTreeV2Header {
         // Validate header checksum
         #[cfg(feature = "checksum")]
         {
-            ensure_len(file_data, pos, 4)?;
+            w.ensure(pos, 4)?;
-            let stored = LittleEndian::read_u32(&file_data[pos..pos + 4]);
+            let stored = LittleEndian::read_u32(&d[pos..pos + 4]);
-            let computed = crate::checksum::jenkins_lookup3(&file_data[offset..pos]);
+            let computed = crate::checksum::jenkins_lookup3(&d[..pos]);
             if computed != stored {
                 return Err(FormatError::ChecksumMismatch {
                     expected: stored,
@@ -163,7 +180,7 @@ impl BTreeV2Header {
 /// Compute maximum records per node for a given depth level.
 /// leaf: (node_size - overhead) / record_size
 /// internal: depends on pointers
-fn max_records_leaf(node_size: u32, record_size: u16) -> u64 {
+pub(crate) fn max_records_leaf(node_size: u32, record_size: u16) -> u64 {
     // Leaf overhead: signature(4) + version(1) + type(1) + checksum(4) = 10
     let overhead = 10u32;
     if node_size <= overhead || record_size == 0 {
@@ -177,10 +194,17 @@ const MAX_DEPTH: u16 = 64;
 
 /// Take `n` records from the traversal's budget, or refuse the tree.
 fn spend(budget: &mut usize, n: usize) -> Result<(), FormatError> {
-    *budget = budget
+    match budget.checked_sub(n) {
-        .checked_sub(n)
+        Some(left) => {
-        .ok_or(FormatError::NestingDepthExceeded)?;
+            *budget = left;
             Ok(())
+        }
+        None => {
+            // Spent: a walk that goes on after a failure stops here.
+            *budget = 0;
+            Err(FormatError::NestingDepthExceeded)
+        }
+    }
 }
 
 /// Collect all records from a B-tree v2 by traversing from the root.
@@ -189,6 +213,17 @@ pub fn collect_btree_v2_records(
     header: &BTreeV2Header,
     offset_size: u8,
     length_size: u8,
+) -> Result<Vec<BTreeV2Record>, FormatError> {
+    collect_btree_v2_records_in(file_data, header, offset_size, length_size)
+}
+
+/// [`collect_btree_v2_records`] over any [`Storage`]: one bounded read per
+/// node.
+pub fn collect_btree_v2_records_in<S: Storage + ?Sized>(
+    file: &S,
+    header: &BTreeV2Header,
+    offset_size: u8,
+    length_size: u8,
 ) -> Result<Vec<BTreeV2Record>, FormatError> {
     if header.total_records == 0 || header.num_records_in_root == 0 {
         return Ok(Vec::new());
@@ -208,24 +243,25 @@ pub fn collect_btree_v2_records(
     // millions of records from a few kilobytes. Counting against what the
     // file could physically contain bounds that without trusting the
     // header's own `total_records`.
-    let mut budget = file_data.len() / usize::from(header.record_size.max(1));
+    let mut budget = len_usize(file) / usize::from(header.record_size.max(1));
 
     let max_leaf_nrec = max_records_leaf(header.node_size, header.record_size);
 
     if header.depth == 0 {
         // Root is a leaf
         parse_leaf_records(
-            file_data,
+            file,
-            header.root_node_address as usize,
+            to_usize(header.root_node_address)?,
             header.num_records_in_root,
             header.record_size,
+            header.node_size,
         )
     } else {
         // Root is internal; traverse recursively
         let mut records = Vec::new();
         collect_internal_records(
-            file_data,
+            file,
-            header.root_node_address as usize,
+            to_usize(header.root_node_address)?,
             header.num_records_in_root,
             header.depth,
             header.record_size,
@@ -240,36 +276,72 @@ pub fn collect_btree_v2_records(
     }
 }
 
+/// A node's bytes: `want` bytes at `offset` (fewer only at the end of the
+/// file), after checking its 4-byte signature. A node is read in one piece
+/// when it fits in `node_size` (every valid node does); a larger claimed
+/// extent — record counts from a damaged parent — is first checked against
+/// the end of the file, so it costs a read only of bytes the file has.
+/// Bounds errors are the whole-file ones: the signature check needs the
+/// first 6 bytes, then `checks` — `(position, length)` pairs relative to
+/// the node, in the order the parser checks them — must lie in the file.
+fn read_node<'a, S: Storage + ?Sized>(
+    file: &'a S,
+    offset: usize,
+    want: usize,
+    node_size: u32,
+    signature: &[u8; 4],
+    checks: &[(usize, usize)],
+) -> Result<Window<'a>, FormatError> {
+    let one_read = usize::try_from(node_size).unwrap_or(usize::MAX).max(6);
+    let w = Window::read(file, offset as u64, want.min(one_read))?;
+    w.ensure(0, 6)?;
+    if &w.bytes[..4] != signature {
+        return Err(FormatError::InvalidBTreeV2Signature);
+    }
+    if want <= one_read {
+        return Ok(w);
+    }
+    for &(rel, len) in checks {
+        Window::check_extent(file, offset as u64, rel, len)?;
+    }
+    Window::read(file, offset as u64, want)
+}
+
 /// Parse records from a leaf node (signature "BTLF").
-fn parse_leaf_records(
+fn parse_leaf_records<S: Storage + ?Sized>(
-    file_data: &[u8],
+    file: &S,
     offset: usize,
     num_records: u16,
     record_size: u16,
+    node_size: u32,
 ) -> Result<Vec<BTreeV2Record>, FormatError> {
     // signature(4) + version(1) + type(1) = 6 bytes header
-    ensure_len(file_data, offset, 6)?;
+    let pos = 6;
-    if &file_data[offset..offset + 4] != b"BTLF" {
-        return Err(FormatError::InvalidBTreeV2Signature);
-    }
-
-    let pos = offset + 6;
     let rs = record_size as usize;
     let total = (num_records as usize)
         .checked_mul(rs)
         .ok_or(FormatError::UnexpectedEof {
             expected: usize::MAX,
-            available: file_data.len(),
+            available: len_usize(file),
         })?;
-    ensure_len(file_data, pos, total)?;
+    let w = read_node(
+        file,
+        offset,
+        pos + total + 4,
+        node_size,
+        b"BTLF",
+        &[(pos, total)],
+    )?;
+    let d = &w.bytes;
+    w.ensure(pos, total)?;
 
     // Validate checksum: 4 bytes after records + padding
     #[cfg(feature = "checksum")]
     {
         let checksum_pos = pos + total;
-        if file_data.len() >= checksum_pos + 4 {
+        if d.len() >= checksum_pos + 4 {
-            let stored = LittleEndian::read_u32(&file_data[checksum_pos..checksum_pos + 4]);
+            let stored = LittleEndian::read_u32(&d[checksum_pos..checksum_pos + 4]);
-            let computed = crate::checksum::jenkins_lookup3(&file_data[offset..checksum_pos]);
+            let computed = crate::checksum::jenkins_lookup3(&d[..checksum_pos]);
             if computed != stored {
                 return Err(FormatError::ChecksumMismatch {
                     expected: stored,
@@ -283,45 +355,57 @@ fn parse_leaf_records(
     for i in 0..num_records as usize {
         let start = pos + i * rs;
         records.push(BTreeV2Record {
-            data: file_data[start..start + rs].to_vec(),
+            data: d[start..start + rs].to_vec(),
         });
     }
     Ok(records)
 }
 
-/// Recursively collect records from an internal node.
+/// An internal node read from the file: its bytes (from the signature on),
-#[allow(clippy::too_many_arguments, clippy::only_used_in_recursion)]
+/// where its records start, and its children as `(address, record count)`.
-fn collect_internal_records(
+struct InternalNode<'a> {
-    file_data: &[u8],
+    node: Window<'a>,
+    records_start: usize,
+    children: Vec<(u64, u16)>,
+}
+
+impl InternalNode<'_> {
+    /// Record `i`, `rs` bytes long.
+    fn record(&self, i: usize, rs: usize) -> Result<&[u8], FormatError> {
+        let overflow = || FormatError::UnexpectedEof {
+            expected: usize::MAX,
+            available: usize::MAX,
+        };
+        let rec_start = i
+            .checked_mul(rs)
+            .and_then(|o| self.records_start.checked_add(o))
+            .ok_or_else(overflow)?;
+        self.node.ensure(rec_start, rs)?;
+        Ok(&self.node.bytes[rec_start..rec_start + rs])
+    }
+}
+
+/// An internal node's layout: where its records start, and its children as
+/// `(address, record count)`.
+#[allow(clippy::too_many_arguments)]
+fn read_internal_node<S: Storage + ?Sized>(
+    file: &S,
     offset: usize,
     num_records: u16,
     depth: u16,
     record_size: u16,
     node_size: u32,
     offset_size: u8,
-    length_size: u8,
     max_leaf_nrec: u64,
-    budget: &mut usize,
+) -> Result<InternalNode<'_>, FormatError> {
-    out: &mut Vec<BTreeV2Record>,
-) -> Result<(), FormatError> {
-    // signature(4) + version(1) + type(1) = 6
-    ensure_len(file_data, offset, 6)?;
-    if &file_data[offset..offset + 4] != b"BTIN" {
-        return Err(FormatError::InvalidBTreeV2Signature);
-    }
-
     let nr = num_records as usize;
     let rs = record_size as usize;
-    let mut pos = offset + 6;
 
-    // Read all records first
+    // Records first
     let records_total = nr.checked_mul(rs).ok_or(FormatError::UnexpectedEof {
         expected: usize::MAX,
-        available: file_data.len(),
+        available: len_usize(file),
     })?;
-    ensure_len(file_data, pos, records_total)?;
-    let records_start = pos;
-    pos += records_total;
 
     // Child pointer layout, as libhdf5 computes it (H5B2__hdr_init): the
     // child's record count is always encoded in the width needed for a
@@ -344,32 +428,118 @@ fn collect_internal_records(
 
     let num_children = nr + 1;
     let child_ptr_size = offset_size as usize + nrec_width + total_nrec_width;
-    ensure_len(file_data, pos, num_children * child_ptr_size)?;
+    let pointers = num_children * child_ptr_size;
+
+    // signature(4) + version(1) + type(1) = 6, records, pointers, checksum.
+    let w = read_node(
+        file,
+        offset,
+        6 + records_total + pointers + 4,
+        node_size,
+        b"BTIN",
+        &[(6, records_total), (6 + records_total, pointers)],
+    )?;
+    let d = &w.bytes;
+    let mut pos = 6;
+    w.ensure(pos, records_total)?;
+    let records_start = pos;
+    pos += records_total;
+
+    w.ensure(pos, pointers)?;
 
-    // Read child pointers
     let mut children = Vec::with_capacity(num_children);
     for _ in 0..num_children {
-        let addr = read_offset(file_data, pos, offset_size)?;
+        let addr = read_offset(d, pos, offset_size)?;
         pos += offset_size as usize;
-        let child_nrec = read_var_uint(file_data, pos, nrec_width)? as u16;
+        let child_nrec = read_var_uint(d, pos, nrec_width)? as u16;
         pos += nrec_width;
         pos += total_nrec_width; // skip total records in subtree
         children.push((addr, child_nrec));
     }
 
+    // The checksum follows the child pointers and covers the node up to it.
+    // Lookups prune children by the keys in this node, so an unverified
+    // internal node could hide a record without any error: libhdf5 refuses
+    // a mismatch here, and so does this.
+    #[cfg(feature = "checksum")]
+    {
+        w.ensure(pos, 4)?;
+        let stored = LittleEndian::read_u32(&d[pos..pos + 4]);
+        let computed = crate::checksum::jenkins_lookup3(&d[..pos]);
+        if computed != stored {
+            return Err(FormatError::ChecksumMismatch {
+                expected: stored,
+                computed,
+            });
+        }
+    }
+    Ok(InternalNode {
+        node: w,
+        records_start,
+        children,
+    })
+}
+
+/// Recursively collect records from an internal node.
+#[allow(clippy::too_many_arguments, clippy::only_used_in_recursion)]
+fn collect_internal_records<S: Storage + ?Sized>(
+    file: &S,
+    offset: usize,
+    num_records: u16,
+    depth: u16,
+    record_size: u16,
+    node_size: u32,
+    offset_size: u8,
+    length_size: u8,
+    max_leaf_nrec: u64,
+    budget: &mut usize,
+    out: &mut Vec<BTreeV2Record>,
+) -> Result<(), FormatError> {
+    let nr = num_records as usize;
+    let rs = record_size as usize;
+    let node = read_internal_node(
+        file,
+        offset,
+        num_records,
+        depth,
+        record_size,
+        node_size,
+        offset_size,
+        max_leaf_nrec,
+    )?;
+    let child_depth = depth - 1;
+
     // Interleave: child[0], record[0], child[1], record[1], ..., child[nr]
     // We collect child[0] records, then record[0], then child[1], etc.
-    for (i, &(child_addr, child_nrec)) in children.iter().enumerate() {
+    // A child that fails does not stop the walk: the others are still
+    // descended into (their records are dropped with the result), then the
+    // first error is returned, as when stopping there. A storage that
+    // records what it lacks (see `storage::touch`) so learns every node the
+    // walk can reach in one attempt. The record budget is spent as before,
+    // so the walk is no longer than a successful one.
+    let mut failed = None;
+    for (i, &(child_addr, child_nrec)) in node.children.iter().enumerate() {
+        if failed.is_some() && *budget == 0 {
+            // The record budget is spent: the tree is refused, and a walk
+            // over what is left could be as long as the one it bounds.
+            break;
+        }
+        if let Err(e) = (|| -> Result<(), FormatError> {
             if child_depth == 0 {
                 // Before parsing, so a refused tree is not also a large allocation.
                 spend(budget, usize::from(child_nrec))?;
-            let leaf_recs =
+                let leaf_recs = parse_leaf_records(
-                parse_leaf_records(file_data, child_addr as usize, child_nrec, record_size)?;
+                    file,
+                    to_usize(child_addr)?,
+                    child_nrec,
+                    record_size,
+                    node_size,
+                )?;
                 out.extend(leaf_recs);
             } else {
                 collect_internal_records(
-                file_data,
+                    file,
-                child_addr as usize,
+                    to_usize(child_addr)?,
                     child_nrec,
                     child_depth,
                     record_size,
@@ -384,44 +554,153 @@ fn collect_internal_records(
 
             // Add record[i] (except after the last child)
             if i < nr {
-            let rec_offset = i.checked_mul(rs).ok_or(FormatError::UnexpectedEof {
+                let data = node.record(i, rs)?;
-                expected: usize::MAX,
-                available: file_data.len(),
-            })?;
-            let rec_start =
-                records_start
-                    .checked_add(rec_offset)
-                    .ok_or(FormatError::UnexpectedEof {
-                        expected: usize::MAX,
-                        available: file_data.len(),
-                    })?;
-            let rec_end = rec_start
-                .checked_add(rs)
-                .ok_or(FormatError::UnexpectedEof {
-                    expected: usize::MAX,
-                    available: file_data.len(),
-                })?;
-            if rec_end > file_data.len() {
-                return Err(FormatError::UnexpectedEof {
-                    expected: rec_end,
-                    available: file_data.len(),
-                });
-            }
                 spend(budget, 1)?;
                 out.push(BTreeV2Record {
-                data: file_data[rec_start..rec_end].to_vec(),
+                    data: data.to_vec(),
                 });
             }
+            Ok(())
+        })() {
+            failed.get_or_insert(e);
+        }
     }
 
+    match failed {
+        Some(e) => Err(e),
+        None => Ok(()),
+    }
+}
+
+/// The records of a B-tree v2 that fall in one key range, found by
+/// descending the tree instead of reading all of it.
+///
+/// `cmp` places a record relative to the range: `Less` if the record sorts
+/// before it, `Greater` if after, `Equal` if the record is in it. The tree
+/// must be ordered consistently with `cmp`, as libhdf5 orders it (a link or
+/// attribute name index by name hash, so all records with one hash form a
+/// range whatever order their names are in). Only the nodes whose key
+/// interval overlaps the range are read: O(depth) nodes plus those holding
+/// the matches. Matches come in tree order.
+pub fn find_btree_v2_records(
+    file_data: &[u8],
+    header: &BTreeV2Header,
+    offset_size: u8,
+    cmp: &mut dyn FnMut(&[u8]) -> Ordering,
+) -> Result<Vec<BTreeV2Record>, FormatError> {
+    find_btree_v2_records_in(file_data, header, offset_size, cmp)
+}
+
+/// [`find_btree_v2_records`] over any [`Storage`]: one bounded read per
+/// node visited.
+pub fn find_btree_v2_records_in<S: Storage + ?Sized>(
+    file: &S,
+    header: &BTreeV2Header,
+    offset_size: u8,
+    cmp: &mut dyn FnMut(&[u8]) -> Ordering,
+) -> Result<Vec<BTreeV2Record>, FormatError> {
+    if header.total_records == 0 || header.num_records_in_root == 0 {
+        return Ok(Vec::new());
+    }
+    if header.depth > MAX_DEPTH {
+        return Err(FormatError::NestingDepthExceeded);
+    }
+    // As in `collect_btree_v2_records`: a valid tree cannot hold more
+    // records than the file has room for, however its children are shared.
+    let mut budget = len_usize(file) / usize::from(header.record_size.max(1));
+    let max_leaf_nrec = max_records_leaf(header.node_size, header.record_size);
+    let mut out = Vec::new();
+    find_in_node(
+        file,
+        header,
+        to_usize(header.root_node_address)?,
+        header.num_records_in_root,
+        header.depth,
+        offset_size,
+        max_leaf_nrec,
+        cmp,
+        &mut budget,
+        &mut out,
+    )?;
+    Ok(out)
+}
+
+#[allow(clippy::too_many_arguments)]
+fn find_in_node<S: Storage + ?Sized>(
+    file: &S,
+    header: &BTreeV2Header,
+    offset: usize,
+    num_records: u16,
+    depth: u16,
+    offset_size: u8,
+    max_leaf_nrec: u64,
+    cmp: &mut dyn FnMut(&[u8]) -> Ordering,
+    budget: &mut usize,
+    out: &mut Vec<BTreeV2Record>,
+) -> Result<(), FormatError> {
+    spend(budget, usize::from(num_records))?;
+    if depth == 0 {
+        let records = parse_leaf_records(
+            file,
+            offset,
+            num_records,
+            header.record_size,
+            header.node_size,
+        )?;
+        out.extend(
+            records
+                .into_iter()
+                .filter(|r| cmp(&r.data) == Ordering::Equal),
+        );
+        return Ok(());
+    }
+    let rs = usize::from(header.record_size);
+    let node = read_internal_node(
+        file,
+        offset,
+        num_records,
+        depth,
+        header.record_size,
+        header.node_size,
+        offset_size,
+        max_leaf_nrec,
+    )?;
+    let nr = usize::from(num_records);
+    let mut order = Vec::with_capacity(nr);
+    for i in 0..nr {
+        order.push(cmp(node.record(i, rs)?));
+    }
+    // Child `i` holds the keys between record `i - 1` and record `i`: it can
+    // hold a match unless the record before it is already past the range or
+    // the record after it is still before it.
+    for (i, &(child_addr, child_nrec)) in node.children.iter().enumerate() {
+        let after_left = i == 0 || order[i - 1] != Ordering::Greater;
+        let before_right = i == nr || order[i] != Ordering::Less;
+        if after_left && before_right {
+            find_in_node(
+                file,
+                header,
+                to_usize(child_addr)?,
+                child_nrec,
+                depth - 1,
+                offset_size,
+                max_leaf_nrec,
+                cmp,
+                budget,
+                out,
+            )?;
+        }
+        if i < nr && order[i] == Ordering::Equal {
+            out.push(BTreeV2Record {
+                data: node.record(i, rs)?.to_vec(),
+            });
+        }
+    }
     Ok(())
 }
 
 /// Most records a subtree whose root is at `depth` can hold (libhdf5's
-/// `cum_max_nrec`): a leaf holds `max_leaf_nrec`; an internal node at depth
+/// `cum_max_nrec`). See [`node_info`].
-/// `d` holds `max_nrec(d)` records and `max_nrec(d) + 1` subtrees of depth
-/// `d - 1`, where `max_nrec(d)` is what fits in a node once each record is
-/// paired with a child pointer of the width depth `d` needs.
 fn cum_max_records(
     node_size: u32,
     record_size: u16,
@@ -429,24 +708,82 @@ fn cum_max_records(
     max_leaf_nrec: u64,
     depth: u16,
 ) -> u64 {
+    node_info_from_leaf(node_size, record_size, offset_size, max_leaf_nrec, depth)
+        .last()
+        .map_or(max_leaf_nrec, |n| n.cum_max_nrec)
+}
+
+/// Capacity of a B-tree v2 node at one depth, as libhdf5 computes it
+/// (`H5B2__hdr_init`'s `node_info`).
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(crate) struct NodeInfo {
+    /// Most records one node at this depth holds.
+    pub(crate) max_nrec: u64,
+    /// Most records a subtree rooted at this depth holds.
+    pub(crate) cum_max_nrec: u64,
+    /// Bytes a subtree's total record count takes in a pointer to a node
+    /// at this depth (0 for a leaf, whose count is its own).
+    pub(crate) cum_max_nrec_size: usize,
+}
+
+/// Node capacities for depths `0..=depth` (entry `d` for depth `d`): a leaf
+/// holds `max_nrec(0)` records; an internal node at depth `d` holds
+/// `max_nrec(d)` records and `max_nrec(d) + 1` subtrees of depth `d - 1`,
+/// where `max_nrec(d)` is what fits in a node once each record is paired
+/// with a child pointer of the width depth `d` needs (address, the child's
+/// record count in the width a *leaf's* maximum needs, and below the first
+/// internal level the child subtree's total in the width its maximum
+/// needs), with one pointer more than records.
+pub(crate) fn node_info(
+    node_size: u32,
+    record_size: u16,
+    offset_size: u8,
+    depth: u16,
+) -> Vec<NodeInfo> {
+    let max_leaf = max_records_leaf(node_size, record_size);
+    node_info_from_leaf(node_size, record_size, offset_size, max_leaf, depth)
+}
+
+fn node_info_from_leaf(
+    node_size: u32,
+    record_size: u16,
+    offset_size: u8,
+    max_leaf_nrec: u64,
+    depth: u16,
+) -> Vec<NodeInfo> {
     // Internal node overhead: signature(4) + version(1) + type(1) + checksum(4).
     const PREFIX: u64 = 10;
     let nrec_width = bytes_for_max_records(max_leaf_nrec) as u64;
-    let mut cum = max_leaf_nrec;
+    let mut info = Vec::with_capacity(usize::from(depth) + 1);
-    let mut cum_width = 0u64;
+    info.push(NodeInfo {
+        max_nrec: max_leaf_nrec,
+        cum_max_nrec: max_leaf_nrec,
+        cum_max_nrec_size: 0,
+    });
     for d in 1..=depth {
-        let ptr = u64::from(offset_size) + nrec_width + if d > 1 { cum_width } else { 0 };
+        let below = info[usize::from(d) - 1];
+        let ptr = u64::from(offset_size)
+            + nrec_width
+            + if d > 1 {
+                below.cum_max_nrec_size as u64
+            } else {
+                0
+            };
         let max_nrec = u64::from(node_size)
             .saturating_sub(PREFIX)
             .saturating_sub(ptr)
             / (u64::from(record_size) + ptr).max(1);
-        cum = max_nrec
+        let cum = max_nrec
             .saturating_add(1)
-            .saturating_mul(cum)
+            .saturating_mul(below.cum_max_nrec)
             .saturating_add(max_nrec);
-        cum_width = bytes_for_max_records(cum) as u64;
+        info.push(NodeInfo {
+            max_nrec,
+            cum_max_nrec: cum,
+            cum_max_nrec_size: bytes_for_max_records(cum),
+        });
     }
-    cum
+    info
 }
 
 #[cfg(test)]
@@ -536,6 +873,8 @@ mod tests {
             buf.extend_from_slice(&child_nrec.to_le_bytes()[..nrec_width]);
             buf.resize(buf.len() + total_width, 0);
         }
+        let sum = crate::checksum::jenkins_lookup3(&buf);
+        buf.extend_from_slice(&sum.to_le_bytes());
         buf
     }
 

crates/clawhdf5-format/src/btree_v2_write.rs

@@ -0,0 +1,503 @@
+//! Writing version-2 B-trees: a header (`BTHD`) and its nodes, leaves
+//! (`BTLF`) and, for more records than one leaf holds, internal nodes
+//! (`BTIN`) to any depth.
+//!
+//! Node capacities come from [`crate::btree_v2::node_info`], the arithmetic
+//! libhdf5 uses (`H5B2__hdr_init`) and the reader decodes pointers with, so
+//! the pointer widths the writer encodes are the ones every reader expects.
+
+use crate::addr::saturating_usize;
+#[cfg(not(feature = "std"))]
+use alloc::{format, vec, vec::Vec};
+
+use crate::btree_v2::{NodeInfo, bytes_for_max_records, node_info};
+use crate::checksum::jenkins_lookup3;
+use crate::error::FormatError;
+
+/// How a B-tree is laid out: its record type and node geometry, as the
+/// header records them.
+#[derive(Debug, Clone, Copy)]
+pub(crate) struct BTreeV2Params {
+    /// Record type (5: link names, 6: link creation order, 8: attribute
+    /// names, 9: attribute creation order, 10/11: chunks).
+    pub(crate) tree_type: u8,
+    /// Bytes per node.
+    pub(crate) node_size: u32,
+    /// Bytes per record.
+    pub(crate) record_size: u16,
+    /// Split and merge percentages. The writer fills nodes itself; these
+    /// only tell libhdf5 when to split and merge as it modifies the tree.
+    pub(crate) split_percent: u8,
+    pub(crate) merge_percent: u8,
+}
+
+/// Size of a B-tree v2 header.
+pub(crate) fn header_size(offset_size: u8, length_size: u8) -> usize {
+    4 + 1 + 1 + 4 + 2 + 2 + 1 + 1 + offset_size as usize + 2 + length_size as usize + 4
+}
+
+/// Deepest tree the writer builds. Even at the smallest fan-out libhdf5's
+/// arithmetic allows, a few levels hold more records than any file could.
+const MAX_WRITE_DEPTH: u16 = 32;
+
+/// Write a B-tree v2 holding `records` (`record_size` bytes each,
+/// concatenated, already in the tree's key order) at `addr`: the header,
+/// then its nodes, each `node_size` bytes. No records gives a header with
+/// an undefined root.
+///
+/// The tree is as shallow as the node size allows: a single leaf when the
+/// records fit one, otherwise internal nodes above leaves. Records are
+/// spread evenly over each node's children, so every node but the root is
+/// at least about half full (above libhdf5's merge threshold, which is below
+/// half), and each node holds at most its depth's maximum.
+pub(crate) fn build_btree_v2(
+    p: BTreeV2Params,
+    records: &[u8],
+    addr: u64,
+    offset_size: u8,
+    length_size: u8,
+) -> Result<Vec<u8>, FormatError> {
+    let rs = usize::from(p.record_size);
+    if rs == 0 || !records.len().is_multiple_of(rs) {
+        return Err(FormatError::SerializationError(format!(
+            "B-tree v2 records are {} bytes, not a multiple of the record size {rs}",
+            records.len()
+        )));
+    }
+    let n = (records.len() / rs) as u64;
+    let hdr_len = header_size(offset_size, length_size);
+
+    // The shallowest depth whose subtree can hold every record.
+    let mut info = node_info(p.node_size, p.record_size, offset_size, 0);
+    let max_leaf = info[0].max_nrec;
+    if max_leaf == 0 || max_leaf > u64::from(u16::MAX) {
+        return Err(FormatError::SerializationError(format!(
+            "a {}-byte B-tree v2 node holds {max_leaf} {}-byte records; \
+             a node holds 1 to 65535",
+            p.node_size, p.record_size
+        )));
+    }
+    let mut depth = 0u16;
+    while info[usize::from(depth)].cum_max_nrec < n {
+        depth += 1;
+        if depth > MAX_WRITE_DEPTH {
+            return Err(FormatError::SerializationError(format!(
+                "{n} records do not fit a B-tree v2 of {}-byte nodes",
+                p.node_size
+            )));
+        }
+        info = node_info(p.node_size, p.record_size, offset_size, depth);
+        let max = info[usize::from(depth)].max_nrec;
+        if max == 0 || max > u64::from(u16::MAX) {
+            return Err(FormatError::SerializationError(format!(
+                "a {}-byte B-tree v2 internal node holds {max} records; \
+                 a node holds 1 to 65535",
+                p.node_size
+            )));
+        }
+    }
+
+    let mut w = TreeWriter {
+        p,
+        records,
+        info: &info,
+        nrec_width: bytes_for_max_records(max_leaf),
+        offset_size,
+        first_node: addr + hdr_len as u64,
+        nodes: Vec::new(),
+    };
+    let root = (n > 0)
+        .then(|| w.node(depth, 0, saturating_usize(n)))
+        .transpose()?;
+
+    let mut out = Vec::with_capacity(hdr_len + w.nodes.len() * p.node_size as usize);
+    out.extend_from_slice(b"BTHD");
+    out.push(0); // version
+    out.push(p.tree_type);
+    out.extend_from_slice(&p.node_size.to_le_bytes());
+    out.extend_from_slice(&p.record_size.to_le_bytes());
+    out.extend_from_slice(&depth.to_le_bytes());
+    out.push(p.split_percent);
+    out.push(p.merge_percent);
+    match root {
+        Some(r) => push_uint(&mut out, r.addr, offset_size as usize),
+        None => out.extend(core::iter::repeat_n(0xFF, offset_size as usize)),
+    }
+    let root_nrec = root.map_or(0, |r| r.nrec);
+    out.extend_from_slice(&(root_nrec as u16).to_le_bytes());
+    push_uint(&mut out, n, length_size as usize);
+    let sum = jenkins_lookup3(&out);
+    out.extend_from_slice(&sum.to_le_bytes());
+    debug_assert_eq!(out.len(), hdr_len);
+    for node in &w.nodes {
+        out.extend_from_slice(node);
+    }
+    Ok(out)
+}
+
+/// A written node, as its parent points at it.
+#[derive(Debug, Clone, Copy)]
+struct NodeRef {
+    addr: u64,
+    /// Records in the node itself.
+    nrec: u64,
+    /// Records in the subtree it roots.
+    all_nrec: u64,
+}
+
+struct TreeWriter<'a> {
+    p: BTreeV2Params,
+    records: &'a [u8],
+    info: &'a [NodeInfo],
+    /// Width of a child's record count: what a leaf's maximum needs.
+    nrec_width: usize,
+    offset_size: u8,
+    /// Address of the first node (right after the header).
+    first_node: u64,
+    /// Nodes in file order (children before their parent).
+    nodes: Vec<Vec<u8>>,
+}
+
+impl TreeWriter<'_> {
+    fn record(&self, i: usize) -> &[u8] {
+        let rs = usize::from(self.p.record_size);
+        &self.records[i * rs..(i + 1) * rs]
+    }
+
+    fn push_node(&mut self, mut node: Vec<u8>) -> u64 {
+        // The checksum covers the node up to it, not the padding after.
+        let sum = jenkins_lookup3(&node);
+        node.extend_from_slice(&sum.to_le_bytes());
+        debug_assert!(node.len() <= self.p.node_size as usize);
+        node.resize(self.p.node_size as usize, 0);
+        let addr = self.first_node + self.nodes.len() as u64 * u64::from(self.p.node_size);
+        self.nodes.push(node);
+        addr
+    }
+
+    /// Write the subtree of `depth` holding records `first..first + n`.
+    fn node(&mut self, depth: u16, first: usize, n: usize) -> Result<NodeRef, FormatError> {
+        let rs = usize::from(self.p.record_size);
+        let mut node = Vec::with_capacity(self.p.node_size as usize);
+        if depth == 0 {
+            debug_assert!(n as u64 <= self.info[0].max_nrec);
+            node.extend_from_slice(b"BTLF");
+            node.push(0); // version
+            node.push(self.p.tree_type);
+            node.extend_from_slice(&self.records[first * rs..(first + n) * rs]);
+            let addr = self.push_node(node);
+            return Ok(NodeRef {
+                addr,
+                nrec: n as u64,
+                all_nrec: n as u64,
+            });
+        }
+
+        // As few children as hold the records, at least two, with the
+        // records spread evenly: `k` children and `k - 1` records between
+        // them.
+        let below = self.info[usize::from(depth) - 1].cum_max_nrec;
+        let k = (n as u64 + 1).div_ceil(below + 1).max(2);
+        let max = self.info[usize::from(depth)].max_nrec;
+        if k - 1 > max || (n as u64) < k - 1 + k {
+            return Err(FormatError::SerializationError(format!(
+                "cannot spread {n} B-tree v2 records over {k} children at depth {depth}"
+            )));
+        }
+        let k = saturating_usize(k);
+        let in_children = n - (k - 1);
+        let (base, extra) = (in_children / k, in_children % k);
+
+        let mut children = Vec::with_capacity(k);
+        let mut separators = Vec::with_capacity(k - 1);
+        let mut next = first;
+        for c in 0..k {
+            let m = base + usize::from(c < extra);
+            children.push(self.node(depth - 1, next, m)?);
+            next += m;
+            if c + 1 < k {
+                separators.push(next);
+                next += 1;
+            }
+        }
+        debug_assert_eq!(next, first + n);
+
+        node.extend_from_slice(b"BTIN");
+        node.push(0); // version
+        node.push(self.p.tree_type);
+        for &s in &separators {
+            node.extend_from_slice(self.record(s));
+        }
+        let total_width = if depth > 1 {
+            self.info[usize::from(depth) - 1].cum_max_nrec_size
+        } else {
+            0
+        };
+        for c in &children {
+            push_uint(&mut node, c.addr, self.offset_size as usize);
+            push_uint(&mut node, c.nrec, self.nrec_width);
+            if depth > 1 {
+                push_uint(&mut node, c.all_nrec, total_width);
+            }
+        }
+        let addr = self.push_node(node);
+        Ok(NodeRef {
+            addr,
+            nrec: (k - 1) as u64,
+            all_nrec: n as u64,
+        })
+    }
+}
+
+/// Append `v` as a `width`-byte little-endian integer.
+fn push_uint(buf: &mut Vec<u8>, v: u64, width: usize) {
+    let bytes = v.to_le_bytes();
+    buf.extend_from_slice(&bytes[..width.min(8)]);
+    buf.extend(vec![0u8; width.saturating_sub(8)]);
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records};
+
+    fn params(node_size: u32, record_size: u16) -> BTreeV2Params {
+        BTreeV2Params {
+            tree_type: 5,
+            node_size,
+            record_size,
+            split_percent: 100,
+            merge_percent: 40,
+        }
+    }
+
+    /// `n` 11-byte records: a big-endian counter, so byte order is key order.
+    fn records(n: usize, rs: usize) -> Vec<u8> {
+        let mut out = Vec::with_capacity(n * rs);
+        for i in 0..n {
+            let mut r = vec![0u8; rs];
+            r[..8].copy_from_slice(&(i as u64).to_be_bytes());
+            out.extend_from_slice(&r);
+        }
+        out
+    }
+
+    fn roundtrip(node_size: u32, rs: u16, n: usize, os: u8, ls: u8) -> BTreeV2Header {
+        let recs = records(n, usize::from(rs));
+        let base = 4096u64;
+        let tree = build_btree_v2(params(node_size, rs), &recs, base, os, ls).unwrap();
+        let mut file = vec![0u8; base as usize];
+        file.extend_from_slice(&tree);
+        let hdr = BTreeV2Header::parse(&file, base as usize, os, ls).unwrap();
+        assert_eq!(hdr.total_records, n as u64);
+        let got = collect_btree_v2_records(&file, &hdr, os, ls).unwrap();
+        assert_eq!(got.len(), n);
+        let flat: Vec<u8> = got.into_iter().flat_map(|r| r.data).collect();
+        assert_eq!(flat, recs, "node {node_size} rs {rs} n {n}");
+        hdr
+    }
+
+    #[test]
+    fn one_leaf_then_deeper_trees_read_back_in_order() {
+        // 512-byte nodes of 11-byte records: 45 per leaf, 1149 at depth 1,
+        // 26 449 at depth 2.
+        let info = node_info(512, 11, 8, 3);
+        assert_eq!(
+            info.iter().map(|i| i.cum_max_nrec).collect::<Vec<_>>(),
+            [45, 1149, 26_449, 608_349]
+        );
+        for (n, depth) in [
+            (0, 0),
+            (1, 0),
+            (45, 0),
+            (46, 1),
+            (1149, 1),
+            (1150, 2),
+            (26_449, 2),
+            (26_450, 3),
+            (100_000, 3),
+        ] {
+            let hdr = roundtrip(512, 11, n, 8, 8);
+            assert_eq!(hdr.depth, depth, "{n} records");
+        }
+    }
+
+    #[test]
+    fn pointer_widths_follow_the_offset_and_length_sizes() {
+        for (os, ls) in [(4, 4), (8, 4), (4, 8), (2, 2)] {
+            roundtrip(512, 11, 5000, os, ls);
+        }
+        // Wide counts: a leaf of 2048 bytes / 9-byte records (226, one byte)
+        // and deeper subtree totals of three bytes.
+        roundtrip(2048, 9, 300_000, 8, 8);
+    }
+
+    #[test]
+    fn every_node_is_within_its_capacity_and_above_the_merge_threshold() {
+        let rs = 17u16;
+        let n = 70_000usize;
+        let info = node_info(512, rs, 8, 3);
+        let recs = records(n, usize::from(rs));
+        let tree = build_btree_v2(params(512, rs), &recs, 0, 8, 8).unwrap();
+        let hdr_len = header_size(8, 8);
+        let nodes = (tree.len() - hdr_len) / 512;
+        for i in 0..nodes {
+            let node = &tree[hdr_len + i * 512..hdr_len + (i + 1) * 512];
+            let sig = &node[..4];
+            if sig == b"BTLF" {
+                continue; // counts checked through the parents below
+            }
+            assert_eq!(sig, b"BTIN");
+        }
+        // Walk from the header: each child's count within [40%, 100%].
+        let hdr = BTreeV2Header::parse(&tree, 0, 8, 8).unwrap();
+        assert_eq!(hdr.depth, 3);
+        assert!(u64::from(hdr.num_records_in_root) <= info[3].max_nrec);
+        fn walk(tree: &[u8], addr: usize, nrec: usize, depth: usize, info: &[NodeInfo], rs: usize) {
+            if depth == 0 {
+                return;
+            }
+            let nrec_w = bytes_for_max_records(info[0].max_nrec);
+            let tot_w = if depth > 1 {
+                info[depth - 1].cum_max_nrec_size
+            } else {
+                0
+            };
+            let mut pos = addr + 6 + nrec * rs;
+            for _ in 0..=nrec {
+                let a = u64::from_le_bytes(tree[pos..pos + 8].try_into().unwrap()) as usize;
+                pos += 8;
+                let mut c = 0usize;
+                for b in 0..nrec_w {
+                    c |= usize::from(tree[pos + b]) << (8 * b);
+                }
+                pos += nrec_w + tot_w;
+                let max = info[depth - 1].max_nrec as usize;
+                assert!(c <= max && c * 100 > max * 40, "{c} of {max}");
+                walk(tree, a, c, depth - 1, info, rs);
+            }
+        }
+        walk(
+            &tree,
+            hdr.root_node_address as usize,
+            usize::from(hdr.num_records_in_root),
+            3,
+            &info,
+            usize::from(rs),
+        );
+        assert!(nodes > 0);
+    }
+
+    /// Descending to a key range finds exactly the records a full read
+    /// holds in it — runs of equal keys that straddle node boundaries
+    /// included — at every depth, and nothing for keys not in the tree.
+    #[test]
+    fn a_key_range_search_matches_a_full_scan() {
+        use crate::btree_v2::find_btree_v2_records;
+        use core::cmp::Ordering;
+        let rs = 11usize;
+        // Keys 0, 0, 0, 2, 2, 2, 4, ...: runs of three, odd keys missing.
+        for n in [1usize, 45, 46, 1150, 30_000] {
+            let mut recs = Vec::with_capacity(n * rs);
+            for i in 0..n {
+                let mut r = vec![0u8; rs];
+                r[..8].copy_from_slice(&((i / 3 * 2) as u64).to_be_bytes());
+                r[8..].copy_from_slice(&[(i % 3) as u8, 0, 0]);
+                recs.extend_from_slice(&r);
+            }
+            let base = 4096u64;
+            let tree = build_btree_v2(params(512, 11), &recs, base, 8, 8).unwrap();
+            let mut file = vec![0u8; base as usize];
+            file.extend_from_slice(&tree);
+            let hdr = BTreeV2Header::parse(&file, base as usize, 8, 8).unwrap();
+            let all = collect_btree_v2_records(&file, &hdr, 8, 8).unwrap();
+            let key = |r: &[u8]| u64::from_be_bytes(r[..8].try_into().unwrap());
+            let last = key(&all[n - 1].data);
+            let probes = (0..=last + 1).step_by(if n > 1000 { 37 } else { 1 });
+            for k in probes.chain([last, last + 1, u64::MAX]) {
+                let found =
+                    find_btree_v2_records(&file, &hdr, 8, &mut |r: &[u8]| key(r).cmp(&k)).unwrap();
+                let want: Vec<&[u8]> = all
+                    .iter()
+                    .map(|r| r.data.as_slice())
+                    .filter(|r| key(r) == k)
+                    .collect();
+                let got: Vec<&[u8]> = found.iter().map(|r| r.data.as_slice()).collect();
+                assert_eq!(got, want, "n {n} key {k}");
+                assert_eq!(
+                    got.len(),
+                    if k % 2 == 0 && k <= last {
+                        want.len()
+                    } else {
+                        0
+                    }
+                );
+            }
+            // Every record, or none, when the whole tree is in or out of range.
+            let every = find_btree_v2_records(&file, &hdr, 8, &mut |_| Ordering::Equal).unwrap();
+            assert_eq!(every.len(), n);
+            let none = find_btree_v2_records(&file, &hdr, 8, &mut |_| Ordering::Less).unwrap();
+            assert!(none.is_empty());
+        }
+    }
+
+    /// A two-level tree read through a `read_at`-only storage gives what
+    /// the slice gives — records, descents and errors — whole, truncated
+    /// at every length, and with each byte of its nodes flipped, and each
+    /// node costs one read.
+    #[test]
+    fn storage_reads_match_slice_reads() {
+        use crate::btree_v2::{
+            collect_btree_v2_records_in, find_btree_v2_records, find_btree_v2_records_in,
+        };
+        use crate::storage::CountingStorage;
+        let (rs, n, base) = (11usize, 120usize, 64usize);
+        let recs = records(n, rs);
+        let tree = build_btree_v2(params(128, 11), &recs, base as u64, 8, 8).unwrap();
+        let mut whole = vec![0u8; base];
+        whole.extend_from_slice(&tree);
+        let hdr = BTreeV2Header::parse(&whole, base, 8, 8).unwrap();
+        assert!(hdr.depth >= 1, "{hdr:?}");
+        let key = |r: &[u8]| u64::from_be_bytes(r[..8].try_into().unwrap());
+        let mut files = Vec::new();
+        for cut in base..=whole.len() {
+            files.push(whole[..cut].to_vec());
+        }
+        for at in base..whole.len() {
+            let mut bad = whole.clone();
+            bad[at] ^= 0x5a;
+            files.push(bad);
+        }
+        let mut ok = 0;
+        for f in &files {
+            let st = CountingStorage::new(f.clone());
+            let want_h = BTreeV2Header::parse(f, base, 8, 8);
+            let got_h = BTreeV2Header::parse_in(&st, base as u64, 8, 8);
+            assert_eq!(format!("{got_h:?}"), format!("{want_h:?}"));
+            // The nodes of the intact header, over each damaged file.
+            let want = collect_btree_v2_records(f, &hdr, 8, 8);
+            st.reset();
+            let got = collect_btree_v2_records_in(&st, &hdr, 8, 8);
+            assert_eq!(format!("{got:?}"), format!("{want:?}"));
+            if want.is_ok() {
+                ok += 1;
+                assert!(st.reads() <= 1 + n as u64 / 3, "{} reads", st.reads());
+            }
+            for k in [0u64, 7, 60, 119, 500] {
+                let want = find_btree_v2_records(f, &hdr, 8, &mut |r: &[u8]| key(r).cmp(&k));
+                let got = find_btree_v2_records_in(&st, &hdr, 8, &mut |r: &[u8]| key(r).cmp(&k));
+                assert_eq!(format!("{got:?}"), format!("{want:?}"));
+            }
+        }
+        assert!(ok > 1);
+    }
+
+    #[test]
+    fn a_node_too_small_or_too_big_is_an_error() {
+        assert!(build_btree_v2(params(16, 11), &records(1, 11), 0, 8, 8).is_err());
+        // A leaf with room for more than 65 535 records.
+        assert!(build_btree_v2(params(1 << 20, 11), &records(1, 11), 0, 8, 8).is_err());
+        // Records that are not whole.
+        assert!(build_btree_v2(params(512, 11), &[0u8; 12], 0, 8, 8).is_err());
+    }
+}

crates/clawhdf5-format/src/checksum.rs

@@ -14,6 +14,45 @@ pub fn jenkins_lookup3(data: &[u8]) -> u32 {
     hashlittle(data, 0)
 }
 
+/// HDF5's Fletcher-32 checksum, as the Fletcher-32 I/O filter (filter id 3)
+/// stores it after each chunk.
+///
+/// A line-for-line port of `H5_checksum_fletcher32` (H5checksum.c, libhdf5
+/// 1.8 through 1.14): big-endian 16-bit words summed in blocks of 360, each
+/// sum reduced after a block by the ones'-complement fold
+/// `(s & 0xffff) + (s >> 16)` rather than `% 65535`, an odd trailing byte
+/// taken as the high byte of a last word, and a final fold of both sums.
+/// The fold and `% 65535` differ whenever a sum is a non-zero multiple of
+/// 65535: the fold leaves 0xffff where the modulo gives 0, so the two
+/// disagree on about one chunk in 32768 and libhdf5 rejects the other's
+/// checksum. This must stay the only implementation.
+pub fn fletcher32(data: &[u8]) -> u32 {
+    let mut sum1: u32 = 0;
+    let mut sum2: u32 = 0;
+    // 360 words keep both sums inside 32 bits between folds (the bound
+    // libhdf5 uses: after a fold sum1 < 0x10200, so sum2 stays below
+    // 360 * 361 / 2 * 0xffff + 360 * 0x10200 + 0x1fffe < 2^32). The adds wrap
+    // like the C unsigned arithmetic all the same.
+    let (words, odd) = data.as_chunks::<2>();
+    for block in words.chunks(360) {
+        for w in block {
+            sum1 = sum1.wrapping_add((u32::from(w[0]) << 8) | u32::from(w[1]));
+            sum2 = sum2.wrapping_add(sum1);
+        }
+        sum1 = (sum1 & 0xffff) + (sum1 >> 16);
+        sum2 = (sum2 & 0xffff) + (sum2 >> 16);
+    }
+    if let [last] = odd {
+        sum1 = sum1.wrapping_add(u32::from(*last) << 8);
+        sum2 = sum2.wrapping_add(sum1);
+        sum1 = (sum1 & 0xffff) + (sum1 >> 16);
+        sum2 = (sum2 & 0xffff) + (sum2 >> 16);
+    }
+    sum1 = (sum1 & 0xffff) + (sum1 >> 16);
+    sum2 = (sum2 & 0xffff) + (sum2 >> 16);
+    (sum2 << 16) | sum1
+}
+
 /// Compute CRC32 (IEEE / ISO 3309) over data.
 ///
 /// When the `fast-checksum` feature is enabled, this uses hardware CRC32
@@ -207,6 +246,19 @@ fn hashlittle(data: &[u8], initval: u32) -> u32 {
 mod tests {
     use super::*;
 
+    /// Values of libhdf5's `H5_checksum_fletcher32` (h5py 3.x's bundled
+    /// libhdf5, called through ctypes). The first three are sums that are
+    /// multiples of 65535, where `% 65535` gave 0 instead of 0xffff.
+    #[test]
+    fn fletcher32_matches_libhdf5() {
+        assert_eq!(fletcher32(&[0x00, 0x01, 0xff, 0xfe]), 0x0001_ffff);
+        assert_eq!(fletcher32(&[0xff; 720]), 0xffff_ffff);
+        assert_eq!(fletcher32(&[0xff; 721]), 0xff00_ff00);
+        assert_eq!(fletcher32(&[0xff; 1441]), 0xff00_ff00);
+        assert_eq!(fletcher32(&[]), 0);
+        assert_eq!(fletcher32(&[7]), 0x0700_0700);
+    }
+
     #[test]
     fn empty_input() {
         // Empty input should return the initial state after no mixing

crates/clawhdf5-format/src/chunk_cache.rs

@@ -262,6 +262,11 @@ struct CachedChunk {
 struct DatasetEntry {
     /// Chunk coordinate -> ChunkInfo (offset + size in file).
     index: Option<Arc<HashMap<ChunkCoord, ChunkInfo>>>,
+    /// The same chunks in the order the chunk index lists them: what
+    /// [`ChunkCache::chunks_for`] returns, so a cached read walks (and, on a
+    /// damaged file, fails at) the chunks in the same order as an uncached
+    /// one, rather than in hash-map order.
+    ordered: Option<Arc<Vec<ChunkInfo>>>,
     /// Pre-built chunk index for O(1) coordinate lookups.
     chunk_index: Option<Arc<ChunkIndex>>,
     /// Pre-computed chunk layout for fast assembly.
@@ -274,6 +279,7 @@ struct DatasetEntry {
 impl DatasetEntry {
     fn weight(&self) -> usize {
         self.index.as_ref().map_or(0, |m| m.len())
+            + self.ordered.as_ref().map_or(0, |o| o.len())
             + self.chunk_index.as_ref().map_or(0, |c| c.num_chunks())
     }
 }
@@ -562,11 +568,22 @@ impl ChunkCache {
         rank: usize,
         build: impl FnOnce() -> Result<Vec<ChunkInfo>, E>,
     ) -> Result<Vec<ChunkInfo>, E> {
-        Ok(self
+        if let Some(ordered) = self.lock().touch(addr).ordered.clone() {
-            .index_for(addr, rank, build)?
+            return Ok(ordered.as_ref().clone());
-            .values()
+        }
-            .cloned()
+        let chunks = build()?;
-            .collect())
+        let map: HashMap<ChunkCoord, ChunkInfo> = chunks
+            .iter()
+            .map(|ci| (ci.offsets.iter().take(rank).copied().collect(), ci.clone()))
+            .collect();
+        let mut inner = self.lock();
+        let entry = inner.touch(addr);
+        // Another thread may have built this dataset's index meanwhile: keep
+        // the first one, so every reader sees the same order.
+        let ordered = Arc::clone(entry.ordered.get_or_insert_with(|| Arc::new(chunks)));
+        entry.index.get_or_insert_with(|| Arc::new(map));
+        inner.trim_datasets(addr);
+        Ok(ordered.as_ref().clone())
     }
 
     fn index_for<E>(
@@ -580,11 +597,14 @@ impl ChunkCache {
         }
         let chunks = build()?;
         let map: HashMap<ChunkCoord, ChunkInfo> = chunks
-            .into_iter()
+            .iter()
-            .map(|ci| (ci.offsets.iter().take(rank).copied().collect(), ci))
+            .map(|ci| (ci.offsets.iter().take(rank).copied().collect(), ci.clone()))
             .collect();
         let mut inner = self.lock();
         let entry = inner.touch(addr);
+        if entry.index.is_none() {
+            entry.ordered = Some(Arc::new(chunks));
+        }
         let index = Arc::clone(entry.index.get_or_insert_with(|| Arc::new(map)));
         inner.trim_datasets(addr);
         Ok(index)
@@ -879,7 +899,7 @@ mod tests {
 
     fn make_chunk(offsets: Vec<u64>, address: u64, size: u32) -> ChunkInfo {
         ChunkInfo {
-            chunk_size: size,
+            chunk_size: u64::from(size),
             filter_mask: 0,
             offsets,
             address,

crates/clawhdf5-format/src/chunk_grid.rs

@@ -135,6 +135,12 @@ impl ChunkGrid {
         let mut rem = index;
         for p in 0..rank {
             let d = self.order[p];
+            // A zero stride: a later dimension has no chunks (its maximum,
+            // or with none recorded its current extent, is 0), so no slot of
+            // the index is a chunk of the dataset.
+            if self.down[p] == 0 {
+                return None;
+            }
             let scaled = rem / self.down[p];
             rem %= self.down[p];
             if scaled >= self.cur_chunks[d] {
@@ -193,6 +199,28 @@ mod tests {
         assert_eq!(g.offsets(11), Some(vec![2, 3]));
     }
 
+    #[test]
+    fn zero_extent_has_no_chunks() {
+        // No maximum recorded and a zero current dimension: every stride
+        // before it is 0 (this divided by zero).
+        let g = ChunkGrid::fixed_array(&[1, 0], None, &[6, 6]).unwrap();
+        for i in 0..16 {
+            assert_eq!(g.offsets(i), None);
+        }
+        let g = ChunkGrid::fixed_array(&[0, 0, 3], Some(&[4, 0, 3]), &[2, 2, 3]).unwrap();
+        for i in 0..16 {
+            assert_eq!(g.offsets(i), None);
+        }
+        let g = ChunkGrid::extensible_array(&[0, 5], Some(&[u64::MAX, 0]), &[2, 2]).unwrap();
+        for i in 0..16 {
+            assert_eq!(g.offsets(i), None);
+        }
+        // A zero last dimension leaves the other strides alone.
+        let g = ChunkGrid::fixed_array(&[4, 0], Some(&[4, 6]), &[2, 3]).unwrap();
+        assert_eq!(g.offsets(0), None);
+        assert_eq!(g.linear_index(&[1, 1]), 3);
+    }
+
     #[test]
     fn rejects_two_unlimited_dims_after_the_first() {
         assert!(ChunkGrid::fixed_array(&[4, 6], Some(&[u64::MAX, u64::MAX]), &[2, 3]).is_err());

crates/clawhdf5-format/src/chunk_index.rs

@@ -18,6 +18,7 @@ use alloc::collections::BTreeMap;
 #[cfg(feature = "std")]
 use std::collections::HashMap;
 
+use crate::addr::to_usize;
 use crate::chunk_cache::ChunkCoord;
 use crate::chunked_read::ChunkInfo;
 
@@ -108,7 +109,7 @@ pub struct ChunkMapping {
     /// File byte address of the compressed chunk.
     pub file_offset: u64,
     /// Size of the compressed chunk in the file.
-    pub file_size: u32,
+    pub file_size: u64,
     /// Filter mask (0 = all filters applied).
     pub filter_mask: u32,
     /// Pre-computed row-copy operations for assembling this chunk into output.
@@ -167,7 +168,15 @@ impl ChunkLayout {
 
         for (_coord, ci) in index.iter() {
             let coord: ChunkCoord = ci.offsets.iter().take(rank).copied().collect();
-            let chunk_offsets: Vec<usize> = coord.iter().map(|&o| o as usize).collect();
+            // `ds_dims` are `usize`: a chunk at an offset past `usize::MAX`
+            // (only on a 32-bit target) lies outside the dataset.
+            let Ok(chunk_offsets) = coord
+                .iter()
+                .map(|&o| to_usize(o))
+                .collect::<Result<Vec<usize>, _>>()
+            else {
+                continue;
+            };
 
             let copies = if rank == 0 {
                 // Scalar dataset — single copy
@@ -329,7 +338,7 @@ mod tests {
 
     fn make_chunk(offsets: Vec<u64>, address: u64, size: u32) -> ChunkInfo {
         ChunkInfo {
-            chunk_size: size,
+            chunk_size: u64::from(size),
             filter_mask: 0,
             offsets,
             address,

crates/clawhdf5-format/src/data_layout.rs

@@ -6,7 +6,9 @@ use alloc::{format, string::String, vec::Vec};
 #[cfg(feature = "std")]
 use std::string::String;
 
+use crate::addr::to_usize;
 use crate::error::FormatError;
+use crate::storage::Storage;
 
 /// A single VDS (Virtual Dataset) source mapping.
 ///
@@ -207,7 +209,7 @@ pub fn parse_vds_mappings(
                     "VDS mapping shares a name with a later entry".into(),
                 ));
             }
-            Ok(idx as usize)
+            to_usize(idx)
         };
 
         let source_file = if flags & VDS_SOURCE_SAME_FILE != 0 {
@@ -309,6 +311,16 @@ impl DataLayout {
         &mut self,
         file_data: &[u8],
         length_size: u8,
+    ) -> Result<(), FormatError> {
+        self.resolve_vds_mappings_in(file_data, length_size)
+    }
+
+    /// [`Self::resolve_vds_mappings`] over any [`Storage`]: one read of the
+    /// global heap collection holding the mappings.
+    pub fn resolve_vds_mappings_in<S: Storage + ?Sized>(
+        &mut self,
+        file_data: &S,
+        length_size: u8,
     ) -> Result<(), FormatError> {
         if let DataLayout::Virtual {
             global_heap_address,
@@ -318,11 +330,8 @@ impl DataLayout {
         } = self
             && let Some(addr) = *global_heap_address
         {
-            let coll = crate::global_heap::GlobalHeapCollection::parse(
+            let coll =
-                file_data,
+                crate::global_heap::GlobalHeapCollection::parse_in(file_data, addr, length_size)?;
-                addr as usize,
-                length_size,
-            )?;
             let obj = coll.get_object(*global_heap_index as u16).ok_or(
                 FormatError::GlobalHeapObjectNotFound {
                     collection_address: addr,
@@ -1305,4 +1314,44 @@ mod tests {
         let blob = [0x01u8, 0, 0, 0, 0, 0, 0, 0, 0];
         assert!(parse_vds_mappings(&blob, 8).unwrap().is_empty());
     }
+
+    /// A virtual dataset's mappings resolve identically through a
+    /// read_at-only CountingStorage, in two reads of the global heap.
+    #[test]
+    fn vds_mappings_through_storage_match_slice() {
+        use crate::message_type::MessageType;
+        use crate::object_header::ObjectHeader;
+        use crate::storage::CountingStorage;
+        let file: &[u8] = include_bytes!("../tests/fixtures/vds_same_file.h5");
+        let sb = crate::superblock::Superblock::parse(file, 0).unwrap();
+        let (os, ls) = (sb.offset_size, sb.length_size);
+        let storage = CountingStorage::new(file.to_vec());
+        let mut virtuals = 0;
+        for child in
+            crate::group_v2::resolve_group_children(file, &sb, sb.root_group_address).unwrap()
+        {
+            let h =
+                ObjectHeader::parse(file, child.object_header_address as usize, os, ls).unwrap();
+            let Some(msg) = h
+                .messages
+                .iter()
+                .find(|m| m.msg_type == MessageType::DataLayout)
+            else {
+                continue;
+            };
+            let mut want = DataLayout::parse(&msg.data, os, ls).unwrap();
+            if !matches!(want, DataLayout::Virtual { .. }) {
+                continue;
+            }
+            let mut got = want.clone();
+            want.resolve_vds_mappings(file, ls).unwrap();
+            storage.reset();
+            got.resolve_vds_mappings_in(&storage, ls).unwrap();
+            assert_eq!(format!("{got:?}"), format!("{want:?}"));
+            assert!(matches!(&got, DataLayout::Virtual { mappings, .. } if !mappings.is_empty()));
+            assert_eq!(storage.reads(), 2);
+            virtuals += 1;
+        }
+        assert!(virtuals >= 1);
+    }
 }

crates/clawhdf5-format/src/data_read.rs

@@ -1,21 +1,25 @@
 //! Raw data reading and typed conversion for HDF5 datasets.
 
 #[cfg(not(feature = "std"))]
-use alloc::{collections::BTreeMap, format, string::String, vec, vec::Vec};
+use alloc::{borrow::Cow, collections::BTreeMap, format, string::String, vec, vec::Vec};
+#[cfg(feature = "std")]
+use std::borrow::Cow;
 
 #[cfg(feature = "std")]
 use std::collections::BTreeMap;
 
+use crate::addr::to_usize;
 #[cfg(feature = "std")]
 use crate::chunk_cache::ChunkCache;
-use crate::chunked_read::read_chunked_data;
+use crate::chunked_read::read_chunked_data_in;
 #[cfg(feature = "std")]
-use crate::chunked_read::{read_chunked_data_cached, read_chunked_data_indexed};
+use crate::chunked_read::{read_chunked_data_cached_in, read_chunked_data_indexed_in};
 use crate::data_layout::DataLayout;
 use crate::dataspace::Dataspace;
 use crate::datatype::{Datatype, DatatypeByteOrder};
 use crate::error::FormatError;
 use crate::filter_pipeline::FilterPipeline;
+use crate::storage::{Storage, read_exact_at};
 
 /// Checks that `[offset, offset + needed)` fits within `data`, guarding the
 /// addition against `usize` overflow from a crafted near-`usize::MAX` offset.
@@ -32,6 +36,80 @@ fn ensure_len(data: &[u8], offset: usize, needed: usize) -> Result<(), FormatErr
     Ok(())
 }
 
+/// The storage checks libhdf5 makes when it opens a dataset, before any
+/// data is read (`H5D__contig_check`, `H5D__compact_init`), so a dataset
+/// they refuse fails to open, as in libhdf5, instead of opening and
+/// reporting a shape nothing can be read from:
+///
+/// - the element count times the element size must not overflow 64 bits
+///   ("size of dataset's storage overflowed" — `cve-2024-32624`
+///   `/Dset_OBJREF`, 2^62 references of 8 bytes);
+/// - contiguous storage at a defined address must end within the file's
+///   `file_len` bytes (the HDF5 data up to the end of file the superblock
+///   records);
+/// - compact data must be exactly the dataset's size.
+///
+/// Deliberately not refused, unlike libhdf5: an empty contiguous dataset at
+/// a defined address (libhdf5's overflow test `addr + 0 <= addr` refuses
+/// it), which clawhdf5 up to v2.7.0 wrote. Chunked and virtual layouts are
+/// checked when their data is read.
+pub fn check_dataset_storage(
+    layout: &DataLayout,
+    dataspace: &Dataspace,
+    datatype: &Datatype,
+    file_len: u64,
+) -> Result<(), FormatError> {
+    if !matches!(
+        layout,
+        DataLayout::Contiguous { .. } | DataLayout::Compact { .. }
+    ) {
+        return Ok(());
+    }
+    const OVERFLOWED: &str = "size of dataset's storage overflowed";
+    let n = dataspace
+        .checked_num_elements()
+        .map_err(|_| FormatError::InvalidDatasetStorage(OVERFLOWED))?;
+    let data_size = n
+        .checked_mul(u64::from(datatype.type_size()))
+        .ok_or(FormatError::InvalidDatasetStorage(OVERFLOWED))?;
+    match layout {
+        DataLayout::Contiguous {
+            address: Some(address),
+            ..
+        } if address
+            .checked_add(data_size)
+            .is_none_or(|end| end > file_len) =>
+        {
+            Err(FormatError::InvalidDatasetStorage(
+                "invalid dataset size, likely file corruption",
+            ))
+        }
+        DataLayout::Compact { data } if data.len() as u64 != data_size => {
+            Err(FormatError::InvalidDatasetStorage(
+                "bad value from dataset header - size of compact dataset's data buffer \
+                 doesn't match size of dataset data",
+            ))
+        }
+        _ => Ok(()),
+    }
+}
+
+/// How many bytes to read from a contiguous dataset's storage of
+/// `storage_size` bytes (the layout message's size) holding `needed` bytes
+/// of elements. libhdf5 reads the elements' bytes from the start of the
+/// storage and ignores storage past them (`H5D__contig_check` checks only
+/// that the elements fit in the file), so a larger storage reads; one too
+/// small to hold the elements is an error.
+pub fn contiguous_read_len(storage_size: u64, needed: usize) -> Result<usize, FormatError> {
+    if storage_size < needed as u64 {
+        return Err(FormatError::DataSizeMismatch {
+            expected: needed,
+            actual: usize::try_from(storage_size).unwrap_or(usize::MAX),
+        });
+    }
+    Ok(needed)
+}
+
 /// Zero-copy read of contiguous raw data, returning a borrowed slice.
 ///
 /// For contiguous layouts, returns a direct `&[u8]` slice into `file_data`.
@@ -43,7 +121,7 @@ pub fn read_raw_data_zerocopy<'a>(
     dataspace: &Dataspace,
     datatype: &Datatype,
 ) -> Result<Option<&'a [u8]>, FormatError> {
-    let num_elements = dataspace.num_elements() as usize;
+    let num_elements = to_usize(dataspace.num_elements())?;
     let elem_size = datatype.type_size() as usize;
     let expected_size = num_elements.checked_mul(elem_size).ok_or_else(|| {
         FormatError::Overflow(format!(
@@ -54,14 +132,8 @@ pub fn read_raw_data_zerocopy<'a>(
     match layout {
         DataLayout::Contiguous { address, size } => {
             let addr = address.ok_or(FormatError::NoDataAllocated)?;
-            let addr = addr as usize;
+            let addr = to_usize(addr)?;
-            let sz = *size as usize;
+            let sz = contiguous_read_len(*size, expected_size)?;
-            if sz != expected_size {
-                return Err(FormatError::DataSizeMismatch {
-                    expected: expected_size,
-                    actual: sz,
-                });
-            }
             ensure_len(file_data, addr, sz)?;
             Ok(Some(&file_data[addr..addr + sz]))
         }
@@ -80,7 +152,17 @@ pub fn read_raw_data(
     dataspace: &Dataspace,
     datatype: &Datatype,
 ) -> Result<Vec<u8>, FormatError> {
-    read_raw_data_full(file_data, layout, dataspace, datatype, None, 8, 8)
+    read_raw_data_in(file_data, layout, dataspace, datatype)
+}
+
+/// [`read_raw_data`] over any [`Storage`].
+pub fn read_raw_data_in<S: Storage + ?Sized>(
+    file_data: &S,
+    layout: &DataLayout,
+    dataspace: &Dataspace,
+    datatype: &Datatype,
+) -> Result<Vec<u8>, FormatError> {
+    read_raw_data_full_in(file_data, layout, dataspace, datatype, None, 8, 8)
 }
 
 /// Resolves a Virtual Dataset source **file name** (as stored in the mapping,
@@ -102,6 +184,27 @@ pub fn read_raw_data_full(
     pipeline: Option<&FilterPipeline>,
     offset_size: u8,
     length_size: u8,
+) -> Result<Vec<u8>, FormatError> {
+    read_raw_data_full_in(
+        file_data,
+        layout,
+        dataspace,
+        datatype,
+        pipeline,
+        offset_size,
+        length_size,
+    )
+}
+
+/// [`read_raw_data_full`] over any [`Storage`].
+pub fn read_raw_data_full_in<S: Storage + ?Sized>(
+    file_data: &S,
+    layout: &DataLayout,
+    dataspace: &Dataspace,
+    datatype: &Datatype,
+    pipeline: Option<&FilterPipeline>,
+    offset_size: u8,
+    length_size: u8,
 ) -> Result<Vec<u8>, FormatError> {
     read_raw_data_full_impl(
         file_data,
@@ -127,6 +230,30 @@ pub fn read_raw_data_full_with_resolver(
     offset_size: u8,
     length_size: u8,
     resolver: Option<&VdsSourceResolver>,
+) -> Result<Vec<u8>, FormatError> {
+    read_raw_data_full_with_resolver_in(
+        file_data,
+        layout,
+        dataspace,
+        datatype,
+        pipeline,
+        offset_size,
+        length_size,
+        resolver,
+    )
+}
+
+/// [`read_raw_data_full_with_resolver`] over any [`Storage`].
+#[allow(clippy::too_many_arguments)]
+pub fn read_raw_data_full_with_resolver_in<S: Storage + ?Sized>(
+    file_data: &S,
+    layout: &DataLayout,
+    dataspace: &Dataspace,
+    datatype: &Datatype,
+    pipeline: Option<&FilterPipeline>,
+    offset_size: u8,
+    length_size: u8,
+    resolver: Option<&VdsSourceResolver>,
 ) -> Result<Vec<u8>, FormatError> {
     read_raw_data_full_impl(
         file_data,
@@ -141,8 +268,8 @@ pub fn read_raw_data_full_with_resolver(
 }
 
 #[allow(clippy::too_many_arguments)]
-fn read_raw_data_full_impl(
+fn read_raw_data_full_impl<S: Storage + ?Sized>(
-    file_data: &[u8],
+    file_data: &S,
     layout: &DataLayout,
     dataspace: &Dataspace,
     datatype: &Datatype,
@@ -151,7 +278,7 @@ fn read_raw_data_full_impl(
     length_size: u8,
     resolver: Option<&VdsSourceResolver>,
 ) -> Result<Vec<u8>, FormatError> {
-    let num_elements = dataspace.num_elements() as usize;
+    let num_elements = to_usize(dataspace.num_elements())?;
     let elem_size = datatype.type_size() as usize;
     let expected_size = num_elements.checked_mul(elem_size).ok_or_else(|| {
         FormatError::Overflow(format!(
@@ -171,20 +298,19 @@ fn read_raw_data_full_impl(
         }
         DataLayout::Contiguous { address, size } => {
             let addr = address.ok_or(FormatError::NoDataAllocated)?;
-            let addr = addr as usize;
+            let addr = to_usize(addr)?;
-            let sz = *size as usize;
+            let sz = contiguous_read_len(*size, expected_size)?;
-            if sz != expected_size {
+            match read_exact_at(file_data, addr as u64, sz)? {
-                return Err(FormatError::DataSizeMismatch {
+                Cow::Borrowed(bytes) => {
-                    expected: expected_size,
-                    actual: sz,
-                });
-            }
-            ensure_len(file_data, addr, sz)?;
                     let mut out = crate::bulk_alloc::vec_for_bulk(sz);
-            out.extend_from_slice(&file_data[addr..addr + sz]);
+                    out.extend_from_slice(bytes);
                     Ok(out)
                 }
-        DataLayout::Chunked { .. } => read_chunked_data(
+                // Fetched for this read: already the caller's copy.
+                Cow::Owned(out) => Ok(out),
+            }
+        }
+        DataLayout::Chunked { .. } => read_chunked_data_in(
             file_data,
             layout,
             dataspace,
@@ -221,9 +347,34 @@ pub fn read_raw_data_cached(
     offset_size: u8,
     length_size: u8,
     cache: &ChunkCache,
+) -> Result<Vec<u8>, FormatError> {
+    read_raw_data_cached_in(
+        file_data,
+        layout,
+        dataspace,
+        datatype,
+        pipeline,
+        offset_size,
+        length_size,
+        cache,
+    )
+}
+
+/// [`read_raw_data_cached`] over any [`Storage`].
+#[cfg(feature = "std")]
+#[allow(clippy::too_many_arguments)]
+pub fn read_raw_data_cached_in<S: Storage + ?Sized>(
+    file_data: &S,
+    layout: &DataLayout,
+    dataspace: &Dataspace,
+    datatype: &Datatype,
+    pipeline: Option<&FilterPipeline>,
+    offset_size: u8,
+    length_size: u8,
+    cache: &ChunkCache,
 ) -> Result<Vec<u8>, FormatError> {
     match layout {
-        DataLayout::Chunked { .. } => read_chunked_data_cached(
+        DataLayout::Chunked { .. } => read_chunked_data_cached_in(
             file_data,
             layout,
             dataspace,
@@ -233,7 +384,7 @@ pub fn read_raw_data_cached(
             length_size,
             cache,
         ),
-        _ => read_raw_data_full(
+        _ => read_raw_data_full_in(
             file_data,
             layout,
             dataspace,
@@ -262,9 +413,34 @@ pub fn read_raw_data_indexed(
     offset_size: u8,
     length_size: u8,
     cache: &ChunkCache,
+) -> Result<Vec<u8>, FormatError> {
+    read_raw_data_indexed_in(
+        file_data,
+        layout,
+        dataspace,
+        datatype,
+        pipeline,
+        offset_size,
+        length_size,
+        cache,
+    )
+}
+
+/// [`read_raw_data_indexed`] over any [`Storage`].
+#[cfg(feature = "std")]
+#[allow(clippy::too_many_arguments)]
+pub fn read_raw_data_indexed_in<S: Storage + ?Sized>(
+    file_data: &S,
+    layout: &DataLayout,
+    dataspace: &Dataspace,
+    datatype: &Datatype,
+    pipeline: Option<&FilterPipeline>,
+    offset_size: u8,
+    length_size: u8,
+    cache: &ChunkCache,
 ) -> Result<Vec<u8>, FormatError> {
     match layout {
-        DataLayout::Chunked { .. } => read_chunked_data_indexed(
+        DataLayout::Chunked { .. } => read_chunked_data_indexed_in(
             file_data,
             layout,
             dataspace,
@@ -274,7 +450,7 @@ pub fn read_raw_data_indexed(
             length_size,
             cache,
         ),
-        _ => read_raw_data_full(
+        _ => read_raw_data_full_in(
             file_data,
             layout,
             dataspace,
@@ -303,6 +479,30 @@ pub fn read_raw_data_selection(
     offset_size: u8,
     length_size: u8,
     selection: &crate::selection::Selection,
+) -> Result<Vec<u8>, FormatError> {
+    read_raw_data_selection_in(
+        file_data,
+        layout,
+        dataspace,
+        datatype,
+        pipeline,
+        offset_size,
+        length_size,
+        selection,
+    )
+}
+
+/// [`read_raw_data_selection`] over any [`Storage`].
+#[allow(clippy::too_many_arguments)]
+pub fn read_raw_data_selection_in<S: Storage + ?Sized>(
+    file_data: &S,
+    layout: &DataLayout,
+    dataspace: &Dataspace,
+    datatype: &Datatype,
+    pipeline: Option<&FilterPipeline>,
+    offset_size: u8,
+    length_size: u8,
+    selection: &crate::selection::Selection,
 ) -> Result<Vec<u8>, FormatError> {
     use crate::selection::Selection;
 
@@ -312,7 +512,7 @@ pub fn read_raw_data_selection(
     // Read only what the selection's bounding box touches when that is
     // possible; everything below is the decode-everything-then-pick path,
     // kept for the cases `partial_read` declines.
-    if let Some(selected) = crate::partial_read::read_selection(
+    if let Some(selected) = crate::partial_read::read_selection_in(
         file_data,
         layout,
         dataspace,
@@ -327,7 +527,7 @@ pub fn read_raw_data_selection(
 
     match selection {
         Selection::All => {
-            return read_raw_data_full(
+            return read_raw_data_full_in(
                 file_data,
                 layout,
                 dataspace,
@@ -347,7 +547,7 @@ pub fn read_raw_data_selection(
     match layout {
         DataLayout::Compact { .. } | DataLayout::Contiguous { .. } => {
             // Read all data, then extract the selection
-            let full_data = read_raw_data_full(
+            let full_data = read_raw_data_full_in(
                 file_data,
                 layout,
                 dataspace,
@@ -371,7 +571,7 @@ pub fn read_raw_data_selection(
             // implicit-index generator, which then indexed past the rank and
             // panicked — only to decode the full dataset anyway.
             crate::chunked_read::chunk_geometry(chunk_dimensions, *version, dataspace, elem_size)?;
-            let full_data = read_raw_data_full(
+            let full_data = read_raw_data_full_in(
                 file_data,
                 layout,
                 dataspace,
@@ -384,7 +584,7 @@ pub fn read_raw_data_selection(
         }
         DataLayout::Virtual { .. } => {
             // Assemble the full virtual dataset, then apply the read selection.
-            let full_data = read_raw_data_full(
+            let full_data = read_raw_data_full_in(
                 file_data,
                 layout,
                 dataspace,
@@ -408,8 +608,8 @@ pub fn read_raw_data_selection(
 /// would report differently from the stored dataspace (unlimited mappings).
 /// Use [`crate::vds::read_virtual_dataset`] to read those.
 #[allow(clippy::too_many_arguments)]
-fn read_virtual_data(
+fn read_virtual_data<S: Storage + ?Sized>(
-    file_data: &[u8],
+    file_data: &S,
     layout: &DataLayout,
     dataspace: &Dataspace,
     datatype: &Datatype,
@@ -420,7 +620,7 @@ fn read_virtual_data(
     let wrapped =
         resolver.map(|r| move |name: &str| -> Result<Option<Vec<u8>>, FormatError> { Ok(r(name)) });
     let wrapped_ref = wrapped.as_ref().map(|w| w as &crate::vds::VdsFileResolver);
-    let v = crate::vds::read_virtual_dataset(
+    let v = crate::vds::read_virtual_dataset_in(
         file_data,
         layout,
         dataspace,
@@ -520,7 +720,7 @@ pub fn extract_selection_from_buffer(
             let rank = dims.len();
             let mut ds_strides = vec![1usize; rank];
             for i in (0..rank.saturating_sub(1)).rev() {
-                ds_strides[i] = ds_strides[i + 1] * dims[i + 1] as usize;
+                ds_strides[i] = ds_strides[i + 1] * to_usize(dims[i + 1])?;
             }
 
             let mut output = Vec::with_capacity(pts.len() * elem_size);
@@ -528,8 +728,8 @@ pub fn extract_selection_from_buffer(
                 let flat: usize = pt
                     .iter()
                     .zip(ds_strides.iter())
-                    .map(|(&p, &s)| p as usize * s)
+                    .map(|(&p, &s)| Ok(to_usize(p)? * s))
-                    .sum();
+                    .sum::<Result<usize, FormatError>>()?;
                 let src = flat * elem_size;
                 if src + elem_size <= full_data.len() {
                     output.extend_from_slice(&full_data[src..src + elem_size]);
@@ -615,6 +815,7 @@ fn datatype_name(dt: &Datatype) -> &'static str {
         Datatype::Enumeration { .. } => "Enumeration",
         Datatype::VariableLength { .. } => "VariableLength",
         Datatype::Array { .. } => "Array",
+        Datatype::Complex { .. } => "Complex",
     }
 }
 
@@ -756,6 +957,220 @@ pub fn read_selection_native<T: NativeElement>(
     crate::gather::gather::<T>(raw, dims, elem_size, selection).map(Some)
 }
 
+/// [`read_selection_native`] of a contiguous dataset in any [`Storage`],
+/// reading only the selected elements' runs (adjacent ones merged, one
+/// [`Storage::read_ranges`] call) instead of the whole dataset.
+///
+/// `Ok(None)` wherever the in-memory fast path does not apply and the
+/// caller converts through the byte readers instead: `datatype` is not
+/// `T`'s native representation, the layout is not contiguous, or the
+/// dataset's bytes cannot be located in the file (no address, storage too
+/// small, past the end of file: the cases [`read_raw_data_zerocopy`]
+/// fails). Otherwise the result and errors are [`read_selection_native`]'s
+/// over those bytes.
+pub fn read_selection_native_in<T: NativeElement, S: Storage + ?Sized>(
+    file_data: &S,
+    layout: &DataLayout,
+    dataspace: &Dataspace,
+    datatype: &Datatype,
+    selection: &crate::selection::Selection,
+) -> Result<Option<Vec<T>>, FormatError> {
+    if !T::is_native(datatype) {
+        return Ok(None);
+    }
+    let DataLayout::Contiguous {
+        address: Some(address),
+        size,
+    } = layout
+    else {
+        return Ok(None);
+    };
+    // Where the dataset's bytes are, as `read_raw_data_zerocopy` finds them.
+    let located = to_usize(dataspace.num_elements())
+        .ok()
+        .and_then(|n| n.checked_mul(datatype.type_size() as usize))
+        .filter(|&len| contiguous_read_len(*size, len).is_ok())
+        .filter(|&len| {
+            address
+                .checked_add(len as u64)
+                .is_some_and(|end| end <= file_data.len())
+        });
+    let Some(len) = located else {
+        return Ok(None);
+    };
+    if let Some(all) = file_data.as_contiguous() {
+        let start = to_usize(*address)?;
+        return read_selection_native(
+            &all[start..start + len],
+            &dataspace.dimensions,
+            datatype,
+            selection,
+        );
+    }
+    let dims = &dataspace.dimensions;
+    let elem_size = core::mem::size_of::<T>();
+    let total = dims
+        .iter()
+        .try_fold(1u64, |acc, &d| acc.checked_mul(d))
+        .ok_or_else(|| FormatError::Overflow("dataset shape overflows".into()))?;
+    let expected = crate::chunked_read::checked_byte_len(total, elem_size)?;
+    if len != expected {
+        return Err(FormatError::DataSizeMismatch {
+            expected,
+            actual: len,
+        });
+    }
+    let bytes = if let crate::selection::Selection::All = selection {
+        read_exact_at(file_data, *address, len)?.into_owned()
+    } else {
+        crate::partial_read::validate(selection, dims)?;
+        crate::gather::gather_storage(file_data, *address, len, dims, elem_size, selection)?
+    };
+    Ok(Some(native_to_vec(&bytes, bytes.len() / elem_size)))
+}
+
+/// The bytes of a slice of [`NativeElement`]s.
+#[cfg(feature = "std")]
+fn bytes_of_mut<T: NativeElement>(values: &mut [T]) -> &mut [u8] {
+    // SAFETY: `T: NativeElement` has no padding and every bit pattern is a
+    // valid value, so its storage may be viewed, and written, as bytes; the
+    // byte slice covers exactly the values' storage and borrows it
+    // exclusively for its lifetime.
+    unsafe {
+        core::slice::from_raw_parts_mut(
+            values.as_mut_ptr().cast::<u8>(),
+            core::mem::size_of_val(values),
+        )
+    }
+}
+
+/// `count` zeroed values of `T`, from zeroed pages where the allocator can
+/// (see [`crate::chunked_read::alloc_output`]) and backed by huge pages when
+/// large. A size taken from the file surfaces as an error, not an abort.
+#[cfg(feature = "std")]
+fn alloc_zeroed_values<T: NativeElement>(count: usize) -> Result<Vec<T>, FormatError> {
+    if count == 0 || core::mem::size_of::<T>() == 0 {
+        return Ok(Vec::new());
+    }
+    let failed = || {
+        FormatError::Overflow(format!(
+            "cannot allocate {count} values of {} bytes for dataset output",
+            core::mem::size_of::<T>()
+        ))
+    };
+    let layout = core::alloc::Layout::array::<T>(count).map_err(|_| failed())?;
+    // SAFETY: `layout` has non-zero size (count > 0, T not zero-sized).
+    let ptr = unsafe { std::alloc::alloc_zeroed(layout) };
+    if ptr.is_null() {
+        return Err(failed());
+    }
+    crate::bulk_alloc::advise_huge_pages(ptr, layout.size());
+    // SAFETY: allocated by the global allocator with the layout of
+    // `[T; count]`, which is what `Vec<T>` with capacity `count` frees; all
+    // bytes are zero, a valid `T` (`NativeElement`: any bit pattern is).
+    Ok(unsafe { Vec::from_raw_parts(ptr.cast::<T>(), count, count) })
+}
+
+/// Read a whole chunked dataset that stores `T` natively
+/// ([`NativeElement::is_native`]) straight into a `Vec<T>`: each chunk is
+/// decoded and copied to its place in the typed output, with no byte buffer
+/// to convert from afterwards. Unallocated chunks read as the dataset's fill
+/// value, as [`crate::fill_value::read_full_with_fill`] makes them.
+///
+/// `Ok(None)` when this does not apply — the datatype is not `T`'s native
+/// representation (another type, another byte order: the caller converts
+/// through the byte readers and the `read_as_*` functions), the layout is
+/// not chunked, no storage is allocated, or the data lives in external
+/// files. `cache` is the file's chunk cache, used as
+/// [`crate::chunked_read::read_chunked_data_cached`] uses it.
+#[cfg(feature = "std")]
+#[allow(clippy::too_many_arguments)]
+pub fn read_chunked_native<T: NativeElement>(
+    messages: &[crate::object_header::HeaderMessage],
+    file_data: &[u8],
+    layout: &DataLayout,
+    dataspace: &Dataspace,
+    datatype: &Datatype,
+    pipeline: Option<&FilterPipeline>,
+    offset_size: u8,
+    length_size: u8,
+    cache: Option<&ChunkCache>,
+) -> Result<Option<Vec<T>>, FormatError> {
+    read_chunked_native_in(
+        messages,
+        file_data,
+        layout,
+        dataspace,
+        datatype,
+        pipeline,
+        offset_size,
+        length_size,
+        cache,
+    )
+}
+
+/// [`read_chunked_native`] over any [`Storage`].
+#[cfg(feature = "std")]
+#[allow(clippy::too_many_arguments)]
+pub fn read_chunked_native_in<T: NativeElement, S: Storage + ?Sized>(
+    messages: &[crate::object_header::HeaderMessage],
+    file_data: &S,
+    layout: &DataLayout,
+    dataspace: &Dataspace,
+    datatype: &Datatype,
+    pipeline: Option<&FilterPipeline>,
+    offset_size: u8,
+    length_size: u8,
+    cache: Option<&ChunkCache>,
+) -> Result<Option<Vec<T>>, FormatError> {
+    use crate::fill_value;
+    use crate::message_type::MessageType;
+
+    if !T::is_native(datatype)
+        || !matches!(layout, DataLayout::Chunked { .. })
+        || !fill_value::has_storage(layout)
+        || messages
+            .iter()
+            .any(|m| m.msg_type == MessageType::ExternalDataFiles)
+    {
+        return Ok(None);
+    }
+    let size = core::mem::size_of::<T>();
+    let mut values = crate::chunked_read::read_chunked_full(
+        file_data,
+        layout,
+        dataspace,
+        datatype,
+        pipeline,
+        offset_size,
+        length_size,
+        cache,
+        |total_bytes| {
+            if !total_bytes.is_multiple_of(size) {
+                return Err(FormatError::DataSizeMismatch {
+                    expected: total_bytes.next_multiple_of(size),
+                    actual: total_bytes,
+                });
+            }
+            alloc_zeroed_values::<T>(total_bytes / size)
+        },
+        |values| bytes_of_mut(values),
+    )?;
+    let fill =
+        fill_value::dataset_fill_value_from_storage(file_data, messages, offset_size, length_size)?;
+    fill_value::apply_to_unallocated_chunks_in(
+        bytes_of_mut(&mut values),
+        file_data,
+        layout,
+        dataspace,
+        size,
+        fill.as_deref(),
+        offset_size,
+        length_size,
+    )?;
+    Ok(Some(values))
+}
+
 /// Convert raw bytes to `f64` values.
 pub fn read_as_f64(raw: &[u8], datatype: &Datatype) -> Result<Vec<f64>, FormatError> {
     // Array datatypes read as a flat sequence of their base elements, and
@@ -1005,9 +1420,11 @@ pub fn read_as_f32(raw: &[u8], datatype: &Datatype) -> Result<Vec<f32>, FormatEr
             result.push(match format {
                 FloatFormat::Single => read_f32_bytes(chunk, &order),
                 FloatFormat::Half => read_f16_bytes(chunk, &order),
-                // Double rounds; every other supported layout (bfloat16, FP8)
+                // Double rounds.
-                // is exact in f32.
+                FloatFormat::Double => format.decode(chunk, &order) as f32,
-                _ => format.decode(chunk, &order) as f32,
+                // Every other supported layout (bfloat16, FP8, FP6, FP4) is
+                // exact in f32.
+                FloatFormat::Other(_) => narrow_decoded(format.decode(chunk, &order)),
             });
         }
         return Ok(result);
@@ -1150,6 +1567,9 @@ pub fn read_compound_fields(
     datatype: &Datatype,
 ) -> Result<Vec<CompoundFieldData>, FormatError> {
     match datatype {
+        Datatype::Complex { size, base_type } => {
+            read_compound_fields(raw, &Datatype::complex_as_compound(*size, base_type))
+        }
         Datatype::Compound { size, members } => {
             let elem_size = *size as usize;
             if elem_size == 0 {
@@ -1165,7 +1585,7 @@ pub fn read_compound_fields(
             let mut fields = Vec::with_capacity(members.len());
             for m in members {
                 let field_size = m.datatype.type_size() as usize;
-                let offset = m.byte_offset as usize;
+                let offset = to_usize(m.byte_offset)?;
                 if offset
                     .checked_add(field_size)
                     .is_none_or(|end| end > elem_size)
@@ -1533,7 +1953,12 @@ enum FloatFormat {
     Double,
     /// Any other IEEE-style layout (implied leading mantissa bit, all-ones
     /// exponent for infinity/NaN) whose values are all exact in `f64`:
-    /// bfloat16, the FP8 formats, and similar.
+    /// bfloat16, FP8 E4M3/E5M2, FP6 E2M3/E3M2, FP4 E2M1, and similar.
+    ///
+    /// libhdf5 (checked against 2.2.0) decodes all of them this way, also the
+    /// OCP MX formats whose specification has no infinity (FP6, FP4) or a
+    /// single NaN (FP8 E4M3): an all-ones exponent is infinity or NaN, not a
+    /// finite value. clawhdf5 follows libhdf5 so both read a file alike.
     Other(FloatLayout),
 }
 
@@ -1632,7 +2057,9 @@ impl FloatLayout {
             if mantissa == 0 {
                 f64::INFINITY
             } else {
-                f64::NAN
+                // The NaN libhdf5 converts every NaN to: all mantissa bits
+                // set (the sign is applied below).
+                LIBHDF5_NAN
             }
         } else {
             let bias = i64::from(self.exponent_bias);
@@ -1653,6 +2080,27 @@ impl FloatLayout {
     }
 }
 
+/// The `f64` NaN libhdf5's conversion (`H5T__conv_f_f`) produces from a NaN
+/// of a non-native float layout: sign clear, every mantissa bit set.
+const LIBHDF5_NAN: f64 = f64::from_bits(0x7FFF_FFFF_FFFF_FFFF);
+
+/// Narrow a value decoded from a non-native float layout to `f32`. Every such
+/// value is exact in `f32`; a NaN becomes the NaN libhdf5 gives for
+/// `H5T_NATIVE_FLOAT` (sign kept, every mantissa bit set) rather than
+/// whatever payload an `as` cast leaves.
+fn narrow_decoded(value: f64) -> f32 {
+    if value.is_nan() {
+        let sign = if value.is_sign_negative() {
+            1u32 << 31
+        } else {
+            0
+        };
+        f32::from_bits(sign | 0x7FFF_FFFF)
+    } else {
+        value as f32
+    }
+}
+
 /// `x * 2^power` without `std` (no `powi`/`libm`). `x` is a non-negative
 /// integer below 2^53, so it is exact.
 fn scale_by_pow2(x: f64, power: i64) -> f64 {
@@ -2011,6 +2459,37 @@ mod tests {
         assert!(got[4].is_nan());
     }
 
+    #[test]
+    fn fp4_decodes_as_libhdf5_does() {
+        // H5T_FLOAT_F4E2M1: 4 significant bits in a byte; the high bits are
+        // padding. libhdf5 2.2.0 reads 0b0110 as +inf and 0b0111 as NaN
+        // (IEEE-style, although OCP MX FP4 has neither), and returns NaNs
+        // with every mantissa bit set.
+        let fp4 = Datatype::FloatingPoint {
+            size: 1,
+            byte_order: DatatypeByteOrder::LittleEndian,
+            bit_offset: 0,
+            bit_precision: 4,
+            exponent_location: 1,
+            exponent_size: 2,
+            mantissa_location: 0,
+            mantissa_size: 1,
+            exponent_bias: 1,
+        };
+        let raw = [0x01, 0x05, 0xF5, 0x06, 0x0E, 0x07, 0x0F];
+        let got = read_as_f64(&raw, &fp4).unwrap();
+        assert_eq!(
+            &got[..5],
+            &[0.5, 3.0, 3.0, f64::INFINITY, f64::NEG_INFINITY]
+        );
+        assert_eq!(got[5].to_bits(), 0x7FFF_FFFF_FFFF_FFFF);
+        assert_eq!(got[6].to_bits(), 0xFFFF_FFFF_FFFF_FFFF);
+        let got = read_as_f32(&raw, &fp4).unwrap();
+        assert_eq!(&got[..3], &[0.5, 3.0, 3.0]);
+        assert_eq!(got[5].to_bits(), 0x7FFF_FFFF);
+        assert_eq!(got[6].to_bits(), 0xFFFF_FFFF);
+    }
+
     #[test]
     fn full_width_signed_unchanged() {
         // Regression: full-width 32-bit signed must be unaffected.
@@ -2632,6 +3111,71 @@ mod tests {
         assert_eq!(result.unwrap(), &[1.5f32, 2.5, 3.5]);
     }
 
+    /// libhdf5 reads a contiguous dataset's elements from the start of its
+    /// storage and ignores storage past them (cve-2024-32623's scalar
+    /// `/Dset1` has 240 bytes of storage for one 4-byte element). Storage too
+    /// small for the elements is still an error.
+    #[test]
+    fn contiguous_storage_larger_than_the_elements_reads() {
+        let dt = make_f64_le_type();
+        let ds = make_simple_dataspace(&[2]);
+        let mut file_data = vec![0u8; 64];
+        file_data[..8].copy_from_slice(&1.5f64.to_le_bytes());
+        file_data[8..16].copy_from_slice(&2.5f64.to_le_bytes());
+        file_data[16..24].copy_from_slice(&9.0f64.to_le_bytes());
+        let layout = DataLayout::Contiguous {
+            address: Some(0),
+            size: 40,
+        };
+        let raw = read_raw_data(&file_data, &layout, &ds, &dt).unwrap();
+        assert_eq!(raw, file_data[..16]);
+        let zc = read_raw_data_zerocopy(&file_data, &layout, &ds, &dt).unwrap();
+        assert_eq!(zc, Some(&file_data[..16]));
+        let small = DataLayout::Contiguous {
+            address: Some(0),
+            size: 8,
+        };
+        assert!(matches!(
+            read_raw_data(&file_data, &small, &ds, &dt),
+            Err(FormatError::DataSizeMismatch { .. })
+        ));
+    }
+
+    /// `H5D__contig_check` / `H5D__compact_init`, run when a dataset opens.
+    #[test]
+    fn dataset_storage_checks_at_open() {
+        let dt = make_f64_le_type();
+        let contiguous = |address| DataLayout::Contiguous { address, size: 0 };
+        // cve-2024-32624 `/Dset_OBJREF`: 2^62 + 2 elements of 8 bytes.
+        let huge = make_simple_dataspace(&[(1 << 62) + 2]);
+        assert_eq!(
+            check_dataset_storage(&contiguous(None), &huge, &dt, 1 << 20),
+            Err(FormatError::InvalidDatasetStorage(
+                "size of dataset's storage overflowed"
+            ))
+        );
+        let ds = make_simple_dataspace(&[4]);
+        assert!(check_dataset_storage(&contiguous(Some(100)), &ds, &dt, 132).is_ok());
+        assert!(matches!(
+            check_dataset_storage(&contiguous(Some(100)), &ds, &dt, 131),
+            Err(FormatError::InvalidDatasetStorage(_))
+        ));
+        assert!(matches!(
+            check_dataset_storage(&contiguous(Some(u64::MAX - 8)), &ds, &dt, u64::MAX),
+            Err(FormatError::InvalidDatasetStorage(_))
+        ));
+        // Not allocated, and (unlike libhdf5) empty at a defined address.
+        assert!(check_dataset_storage(&contiguous(None), &ds, &dt, 0).is_ok());
+        let empty = make_simple_dataspace(&[0]);
+        assert!(check_dataset_storage(&contiguous(Some(64)), &empty, &dt, 64).is_ok());
+        let compact = |n: usize| DataLayout::Compact { data: vec![0; n] };
+        assert!(check_dataset_storage(&compact(32), &ds, &dt, 0).is_ok());
+        assert!(matches!(
+            check_dataset_storage(&compact(24), &ds, &dt, 0),
+            Err(FormatError::InvalidDatasetStorage(_))
+        ));
+    }
+
     #[test]
     fn zerocopy_size_mismatch() {
         let dt = make_f64_le_type();

crates/clawhdf5-format/src/dataspace.rs

@@ -7,6 +7,9 @@ use alloc::vec::Vec;
 
 use crate::error::FormatError;
 
+/// Most dimensions a dataspace can have (`H5S_MAX_RANK`).
+pub const MAX_RANK: u8 = 32;
+
 /// Type of dataspace.
 #[derive(Debug, Clone, PartialEq)]
 pub enum DataspaceType {
@@ -67,6 +70,12 @@ impl Dataspace {
         let version = data[0];
         let rank = data[1];
         let flags = data[2];
+        // H5O__sdspace_decode's checks.
+        if rank > MAX_RANK {
+            return Err(FormatError::InvalidDataspace(
+                "simple dataspace dimensionality is too large",
+            ));
+        }
 
         let (space_type, header_size) = match version {
             1 => {
@@ -88,6 +97,11 @@ impl Dataspace {
                     2 => DataspaceType::Null,
                     _ => return Err(FormatError::InvalidDataspaceType(type_byte)),
                 };
+                if st != DataspaceType::Simple && rank > 0 {
+                    return Err(FormatError::InvalidDataspace(
+                        "invalid rank for scalar or NULL dataspace",
+                    ));
+                }
                 (st, 4usize)
             }
             _ => return Err(FormatError::InvalidDataspaceVersion(version)),
@@ -107,8 +121,13 @@ impl Dataspace {
         // Read max dimensions if flags bit 0 is set
         let max_dimensions = if flags & 0x01 != 0 {
             let mut max_dims = Vec::with_capacity(rank as usize);
-            for _ in 0..rank {
+            for &dim in &dimensions {
                 let val = read_length(data, pos, length_size)?;
+                if dim > val {
+                    return Err(FormatError::InvalidDataspace(
+                        "dataspace dimension size is greater than its maximum size",
+                    ));
+                }
                 max_dims.push(val);
                 pos += ls;
             }
@@ -176,7 +195,6 @@ impl Dataspace {
         match self.space_type {
             DataspaceType::Null => Ok(0),
             DataspaceType::Scalar => Ok(1),
-            DataspaceType::Simple if self.dimensions.is_empty() => Ok(0),
             DataspaceType::Simple => self
                 .dimensions
                 .iter()
@@ -195,18 +213,14 @@ impl Dataspace {
         match self.space_type {
             DataspaceType::Null => 0,
             DataspaceType::Scalar => 1,
-            DataspaceType::Simple => {
+            // A simple dataspace of rank 0 holds one element, as in libhdf5
-                if self.dimensions.is_empty() {
+            // (the product of no dimensions). Saturate rather than wrap: a
-                    0
+            // wrapped product could under-size a buffer. Size-critical
-                } else {
+            // callers use `checked_num_elements`.
-                    // Saturate rather than wrap: a wrapped product could
+            DataspaceType::Simple => self
-                    // under-size a buffer. Size-critical callers use
+                .dimensions
-                    // `checked_num_elements`.
-                    self.dimensions
                 .iter()
-                        .fold(1u64, |acc, &d| acc.saturating_mul(d))
+                .fold(1u64, |acc, &d| acc.saturating_mul(d)),
-                }
-            }
         }
     }
 }
@@ -352,4 +366,44 @@ mod tests {
         let ds = Dataspace::parse(&data, 8).unwrap();
         assert_eq!(ds.max_dimensions, Some(vec![10]));
     }
+
+    /// A simple dataspace of rank 0 (cve-2020-18494's `/dset1`) holds one
+    /// element in libhdf5, which h5py reads as shape `()`. It was 0.
+    #[test]
+    fn simple_rank_zero_holds_one_element() {
+        let data = build_v2_dataspace(0, 0, 1, &[], None);
+        let ds = Dataspace::parse(&data, 8).unwrap();
+        assert_eq!(ds.space_type, DataspaceType::Simple);
+        assert_eq!(ds.num_elements(), 1);
+        assert_eq!(ds.checked_num_elements().unwrap(), 1);
+    }
+
+    /// `H5O__sdspace_decode`'s checks.
+    #[test]
+    fn refuses_what_libhdf5_refuses() {
+        let too_many = build_v2_dataspace(33, 0, 1, &[1; 33], None);
+        assert!(matches!(
+            Dataspace::parse(&too_many, 8),
+            Err(FormatError::InvalidDataspace(_))
+        ));
+        let scalar_with_rank = build_v2_dataspace(1, 0, 0, &[4], None);
+        assert!(matches!(
+            Dataspace::parse(&scalar_with_rank, 8),
+            Err(FormatError::InvalidDataspace(_))
+        ));
+        let null_with_rank = build_v2_dataspace(1, 0, 2, &[4], None);
+        assert!(matches!(
+            Dataspace::parse(&null_with_rank, 8),
+            Err(FormatError::InvalidDataspace(_))
+        ));
+        let over_max = build_v1_dataspace(2, 0x01, &[5, 20], Some(&[10, 10]));
+        assert!(matches!(
+            Dataspace::parse(&over_max, 8),
+            Err(FormatError::InvalidDataspace(_))
+        ));
+        // 32 dimensions, and a size equal to the maximum or unlimited, are fine.
+        assert!(Dataspace::parse(&build_v2_dataspace(32, 0, 1, &[1; 32], None), 8).is_ok());
+        let at_max = build_v1_dataspace(2, 0x01, &[10, 20], Some(&[10, u64::MAX]));
+        assert!(Dataspace::parse(&at_max, 8).is_ok());
+    }
 }

crates/clawhdf5-format/src/datatype.rs

@@ -140,6 +140,19 @@ pub enum Datatype {
         base_type: Box<Datatype>,
         dimensions: Vec<u32>,
     },
+    /// Class 11: HDF5 2.0 native complex number (`H5T_COMPLEX`, datatype
+    /// message version 5): two consecutive `base_type` values, real then
+    /// imaginary, in rectangular form. `size` is twice the base size and the
+    /// base is an IEEE float.
+    ///
+    /// This variant exists for **writing** (see
+    /// `type_builders::make_native_complex_f64_type`): only libhdf5 2.0 and
+    /// newer can read class 11, so it is opt-in and h5py's compound `{r, i}`
+    /// stays the default complex encoding. [`Datatype::parse`] still
+    /// surfaces a class-11 message as that equivalent `{r, i}` compound, so
+    /// every compound reader handles both encodings; parsing what this
+    /// variant serializes therefore yields a `Compound`, not a `Complex`.
+    Complex { size: u32, base_type: Box<Datatype> },
 }
 
 /// Longest opaque tag that can be stored: its NUL-padded length must fit
@@ -862,19 +875,7 @@ impl Datatype {
                         actual: size as usize,
                     });
                 }
-                let members = vec![
+                Ok((Self::complex_as_compound(size, &base_type), pos))
-                    CompoundMember {
-                        name: String::from("r"),
-                        byte_offset: 0,
-                        datatype: base_type.clone(),
-                    },
-                    CompoundMember {
-                        name: String::from("i"),
-                        byte_offset: base_size as u64,
-                        datatype: base_type,
-                    },
-                ];
-                Ok((Datatype::Compound { size, members }, pos))
             }
             _ => Err(FormatError::InvalidDatatypeClass(class_id)),
         }
@@ -939,7 +940,8 @@ impl Datatype {
                 .try_for_each(|m| m.datatype.check_unused_bits()),
             Datatype::Enumeration { base_type, .. }
             | Datatype::VariableLength { base_type, .. }
-            | Datatype::Array { base_type, .. } => base_type.check_unused_bits(),
+            | Datatype::Array { base_type, .. }
+            | Datatype::Complex { base_type, .. } => base_type.check_unused_bits(),
             _ => Ok(()),
         }
     }
@@ -1046,7 +1048,8 @@ impl Datatype {
                 } else {
                     0
                 };
-                let mut buf = Self::build_header(9, 1, [bf0, bf1, 0], *size);
+                let version = base_type.min_parent_version().max(1);
+                let mut buf = Self::build_header(9, version, [bf0, bf1, 0], *size);
                 buf.extend_from_slice(&base_type.serialize());
                 buf
             }
@@ -1054,7 +1057,11 @@ impl Datatype {
                 let num = members.len() as u16;
                 let bf0 = (num & 0xFF) as u8;
                 let bf1 = ((num >> 8) & 0xFF) as u8;
-                let mut buf = Self::build_header(6, 3, [bf0, bf1, 0], *size);
+                let version = members
+                    .iter()
+                    .map(|m| m.datatype.min_parent_version())
+                    .fold(3, u8::max);
+                let mut buf = Self::build_header(6, version, [bf0, bf1, 0], *size);
                 let ob = offset_bytes_for_size(*size);
                 for m in members {
                     // Null-terminated name
@@ -1097,7 +1104,8 @@ impl Datatype {
                 base_type,
                 dimensions,
             } => {
-                let mut buf = Self::build_header(10, 3, [0, 0, 0], self.type_size());
+                let version = base_type.min_parent_version().max(3);
+                let mut buf = Self::build_header(10, version, [0, 0, 0], self.type_size());
                 buf.push(dimensions.len() as u8);
                 for &d in dimensions {
                     buf.extend_from_slice(&d.to_le_bytes());
@@ -1152,8 +1160,81 @@ impl Datatype {
                 };
                 Self::build_header(7, version, [bf0, 0, 0], *size)
             }
+            Datatype::Complex { size, base_type } => {
+                // Version 5 (HDF5 2.0), as libhdf5's `H5O__dtype_encode_helper`
+                // writes it: bit 0 = homogeneous (the only kind libhdf5
+                // supports), bits 1-2 = form (0, rectangular); the base
+                // datatype message follows.
+                let mut buf = Self::build_header(11, 5, [0x01, 0, 0], *size);
+                buf.extend_from_slice(&base_type.serialize());
+                buf
             }
         }
+    }
+
+    /// The `{r, i}` compound equivalent to a native complex type of `size`
+    /// bytes over `base_type`: `r` at offset 0, `i` right after it — the
+    /// shape h5py writes for numpy complex dtypes, and what [`Self::parse`]
+    /// returns for a class-11 message. Readers that meet a
+    /// [`Datatype::Complex`] handle it through this view.
+    pub fn complex_as_compound(size: u32, base_type: &Datatype) -> Datatype {
+        let base_size = base_type.type_size();
+        Datatype::Compound {
+            size,
+            members: vec![
+                CompoundMember {
+                    name: String::from("r"),
+                    byte_offset: 0,
+                    datatype: base_type.clone(),
+                },
+                CompoundMember {
+                    name: String::from("i"),
+                    byte_offset: u64::from(base_size),
+                    datatype: base_type.clone(),
+                },
+            ],
+        }
+    }
+
+    /// The lowest datatype message version a type that contains this one
+    /// may be encoded with. libhdf5 raises a compound, array, variable-length
+    /// or enum type to the version of its members (`H5O_DTYPE_CHECK_VERSION`
+    /// in `H5Odtype.c`), so a type holding a native complex (version 5) is
+    /// itself written as version 5; everything else we write keeps the
+    /// container's own version.
+    fn min_parent_version(&self) -> u8 {
+        match self {
+            Datatype::Complex { .. } => 5,
+            Datatype::Compound { members, .. } => members
+                .iter()
+                .map(|m| m.datatype.min_parent_version())
+                .fold(0, u8::max),
+            Datatype::Enumeration { base_type, .. }
+            | Datatype::VariableLength { base_type, .. }
+            | Datatype::Array { base_type, .. } => base_type.min_parent_version(),
+            _ => 0,
+        }
+    }
+
+    /// The highest datatype message version in this type's encoding, its
+    /// members' and base types' included (the version decides which HDF5
+    /// releases can read it: 1-3 HDF5 1.8, 4 HDF5 1.12, 5 HDF5 2.0).
+    pub fn max_encoded_version(&self) -> u8 {
+        let own = self.serialize().first().map_or(0, |b| b >> 4);
+        let inner = match self {
+            Datatype::Compound { members, .. } => members
+                .iter()
+                .map(|m| m.datatype.max_encoded_version())
+                .max()
+                .unwrap_or(0),
+            Datatype::Enumeration { base_type, .. }
+            | Datatype::VariableLength { base_type, .. }
+            | Datatype::Array { base_type, .. }
+            | Datatype::Complex { base_type, .. } => base_type.max_encoded_version(),
+            _ => 0,
+        };
+        own.max(inner)
+    }
 
     /// Check that this datatype can be written: every part of it has an
     /// on-disk encoding, and the encoding is one the reader (and libhdf5)
@@ -1189,6 +1270,20 @@ impl Datatype {
             Datatype::Enumeration { base_type, .. }
             | Datatype::VariableLength { base_type, .. }
             | Datatype::Array { base_type, .. } => base_type.check_encodable_parts(),
+            // libhdf5 only builds complex types over IEEE floats
+            // (`H5Tcomplex_create`), always twice the base size.
+            Datatype::Complex { size, base_type } => match base_type.as_ref() {
+                Datatype::FloatingPoint { size: b, .. } if b.checked_mul(2) == Some(*size) => {
+                    Ok(())
+                }
+                Datatype::FloatingPoint { .. } => Err(FormatError::SerializationError(format!(
+                    "complex datatype of size {size} is not twice its base size {}",
+                    base_type.type_size()
+                ))),
+                _ => Err(FormatError::SerializationError(
+                    "complex datatype base must be a floating-point type".into(),
+                )),
+            },
             _ => Ok(()),
         }
     }
@@ -1216,6 +1311,7 @@ impl Datatype {
             Datatype::Reference { size, .. } => *size,
             Datatype::Enumeration { size, .. } => *size,
             Datatype::VariableLength { size, .. } => *size,
+            Datatype::Complex { size, .. } => *size,
             Datatype::Array {
                 base_type,
                 dimensions,
@@ -1815,6 +1911,89 @@ mod tests {
         ));
     }
 
+    #[test]
+    fn native_complex_serializes_as_libhdf5_2_0_does() {
+        use crate::type_builders::{make_native_complex_f32_type, make_native_complex_f64_type};
+        let dt = make_native_complex_f64_type();
+        assert_eq!(dt.serialize(), COMPLEX_F64_HDF5_2_0);
+        assert_eq!(dt.type_size(), 16);
+        dt.check_encodable().unwrap();
+        // Parsing surfaces class 11 as the equivalent `{r, i}` compound.
+        let (parsed, _) = Datatype::parse(&dt.serialize()).unwrap();
+        assert_eq!(
+            parsed,
+            Datatype::complex_as_compound(16, &crate::type_builders::make_f64_type())
+        );
+
+        let f32c = make_native_complex_f32_type().serialize();
+        assert_eq!(
+            &f32c[..8],
+            &[0x5b, 0x01, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00]
+        );
+        assert_eq!(
+            &f32c[8..],
+            &crate::type_builders::make_f32_type().serialize()[..]
+        );
+    }
+
+    #[test]
+    fn compound_holding_native_complex_serializes_as_libhdf5_2_0_does() {
+        // The same type as `test_compound_with_complex_member_from_hdf5_2_0`:
+        // libhdf5 raises the compound to version 5 for its complex member.
+        let dt = Datatype::Compound {
+            size: 24,
+            members: vec![
+                CompoundMember {
+                    name: "z".into(),
+                    byte_offset: 0,
+                    datatype: crate::type_builders::make_native_complex_f64_type(),
+                },
+                CompoundMember {
+                    name: "k".into(),
+                    byte_offset: 16,
+                    datatype: crate::type_builders::make_i64_type(),
+                },
+            ],
+        };
+        let mut want = vec![
+            0x56, 0x02, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, b'z', 0x00, 0x00,
+        ];
+        want.extend_from_slice(&COMPLEX_F64_HDF5_2_0);
+        want.extend_from_slice(&[b'k', 0x00, 0x10]);
+        want.extend_from_slice(&[
+            0x10, 0x08, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00,
+        ]);
+        assert_eq!(dt.serialize(), want);
+
+        // An array of complex is raised to version 5 as well; one without
+        // stays at version 3.
+        let arr = Datatype::Array {
+            base_type: Box::new(crate::type_builders::make_native_complex_f32_type()),
+            dimensions: vec![2],
+        };
+        assert_eq!(arr.serialize()[0], 0x5a);
+        arr.check_encodable().unwrap();
+        let plain = Datatype::Array {
+            base_type: Box::new(crate::type_builders::make_f32_type()),
+            dimensions: vec![2],
+        };
+        assert_eq!(plain.serialize()[0], 0x3a);
+    }
+
+    #[test]
+    fn native_complex_must_be_twice_an_ieee_float() {
+        let bad_size = Datatype::Complex {
+            size: 12,
+            base_type: Box::new(crate::type_builders::make_f64_type()),
+        };
+        assert!(bad_size.check_encodable().is_err());
+        let int_base = Datatype::Complex {
+            size: 8,
+            base_type: Box::new(crate::type_builders::make_i32_type()),
+        };
+        assert!(int_base.check_encodable().is_err());
+    }
+
     #[test]
     fn test_reference_object() {
         let buf = build_dt_header(7, 1, [0, 0, 0], 8);

crates/clawhdf5-format/src/ea_writer.rs

@@ -3,6 +3,7 @@
 #[cfg(not(feature = "std"))]
 extern crate alloc;
 
+use crate::addr::saturating_usize;
 #[cfg(not(feature = "std"))]
 use alloc::{vec, vec::Vec};
 
@@ -17,9 +18,10 @@ pub(crate) fn serialize_v4_extensible_array(
     ea_address: u64,
     offset_size: u8,
     element_size: u32,
+    version: u8,
 ) -> Vec<u8> {
     let mut buf = Vec::new();
-    buf.push(4); // version
+    buf.push(version);
     buf.push(2); // class = chunked
     buf.push(0x00); // flags
 
@@ -86,7 +88,7 @@ pub fn build_extensible_array_at(
     ea_base_address: u64,
 ) -> Vec<u8> {
     let os = offset_size as usize;
-    let chunk_size_bytes = has_filters.then(|| filtered_chunk_size_len(slots));
+    let chunk_size_bytes = has_filters.then(|| filtered_chunk_size_len(slots, length_size));
     let elem_size = os + chunk_size_bytes.map_or(0, |n| n + 4);
     let client_id: u8 = if has_filters { 1 } else { 0 };
     let arr_off_size = (MAX_NELMTS_BITS as usize).div_ceil(8);
@@ -247,7 +249,7 @@ pub fn build_extensible_array_at(
 
     // Header (EAHD). The six statistics are, in order: super blocks, their
     // bytes, data blocks, their bytes, max index set, elements realised.
-    let mut out = Vec::with_capacity((cursor - ea_base_address) as usize);
+    let mut out = Vec::with_capacity(saturating_usize(cursor - ea_base_address));
     out.extend_from_slice(b"EAHD");
     out.push(0); // version
     out.push(client_id);

crates/clawhdf5-format/src/error.rs

@@ -12,7 +12,11 @@ use std::string::String;
 use core::fmt;
 
 /// Errors that can occur when parsing HDF5 binary format structures.
+///
+/// Non-exhaustive: new failure modes (new storage backends, new file
+/// features) add variants, so a `match` needs a wildcard arm.
 #[derive(Debug, Clone, PartialEq, Eq)]
+#[non_exhaustive]
 pub enum FormatError {
     /// The HDF5 magic signature was not found at any valid offset.
     SignatureNotFound,
@@ -163,6 +167,19 @@ pub enum FormatError {
     VlDataError(String),
     /// Serialization error.
     SerializationError(String),
+    /// The file's library version bounds
+    /// ([`FileWriter::libver_bounds`](crate::file_writer::FileWriter::libver_bounds))
+    /// do not allow what was asked for: `what` needs the format of HDF5
+    /// `needs` or later, and the high bound is `high` (or the low bound is
+    /// above the high one, with `needs` the low bound).
+    LibverBound {
+        /// What cannot be written.
+        what: String,
+        /// The oldest release whose format holds it.
+        needs: crate::libver::LibVer,
+        /// The file's high bound.
+        high: crate::libver::LibVer,
+    },
     /// Dataset is missing data.
     DatasetMissingData,
     /// Dataset is missing shape.
@@ -223,6 +240,33 @@ pub enum FormatError {
         /// The file's actual length in bytes.
         actual_len: u64,
     },
+    /// A link libhdf5 refuses to list: a symbol-table entry with an empty
+    /// name ("invalid link name"). Listing the group fails, as in libhdf5.
+    InvalidLinkName,
+    /// A dataspace message libhdf5 refuses to decode (the reason is
+    /// libhdf5's own error text): more than 32 dimensions, a rank on a
+    /// scalar or null dataspace, a dimension larger than its maximum.
+    InvalidDataspace(&'static str),
+    /// A dataset whose storage libhdf5 refuses when it opens the dataset
+    /// (the reason is libhdf5's own error text): an element count times
+    /// element size that overflows, contiguous storage past the end of the
+    /// file, compact data of the wrong size.
+    InvalidDatasetStorage(&'static str),
+    /// A superblock extension message libhdf5 refuses to decode when it
+    /// opens the file (the reason is libhdf5's own error text): a File Space
+    /// Info message that runs off its end or has a bad page size, a metadata
+    /// cache image outside the file, …
+    InvalidSuperblockExtension(&'static str),
+    /// A metadata cache image block libhdf5 refuses to load (the reason is
+    /// libhdf5's own error text).
+    InvalidCacheImage(&'static str),
+    /// The [`Storage`](crate::storage::Storage) backend failed to serve a
+    /// read (an I/O or network error, or a short read inside the file).
+    Storage(String),
+    /// The operation still needs the whole file as one slice and the
+    /// [`Storage`](crate::storage::Storage) backend has no contiguous view
+    /// (`as_contiguous()` is `None`); the text names the operation.
+    ContiguousStorageRequired(&'static str),
 }
 
 impl fmt::Display for FormatError {
@@ -419,6 +463,13 @@ impl fmt::Display for FormatError {
             FormatError::SerializationError(msg) => {
                 write!(f, "serialization error: {msg}")
             }
+            FormatError::LibverBound { what, needs, high } => {
+                write!(
+                    f,
+                    "{what} needs the HDF5 {needs} file format, above the high \
+                     library version bound ({high})"
+                )
+            }
             FormatError::DatasetMissingData => {
                 write!(f, "dataset is missing data")
             }
@@ -494,6 +545,31 @@ impl fmt::Display for FormatError {
                      but the file is {actual_len} bytes"
                 )
             }
+            FormatError::InvalidLinkName => {
+                write!(f, "invalid link name: a group entry has an empty name")
+            }
+            FormatError::InvalidDataspace(why) => {
+                write!(f, "invalid dataspace: {why}")
+            }
+            FormatError::InvalidDatasetStorage(why) => {
+                write!(f, "invalid dataset storage: {why}")
+            }
+            FormatError::InvalidSuperblockExtension(why) => {
+                write!(f, "invalid superblock extension: {why}")
+            }
+            FormatError::InvalidCacheImage(why) => {
+                write!(f, "invalid metadata cache image: {why}")
+            }
+            FormatError::Storage(why) => {
+                write!(f, "storage read failed: {why}")
+            }
+            FormatError::ContiguousStorageRequired(what) => {
+                write!(
+                    f,
+                    "{what} needs the whole file in memory, which this storage backend does \
+                     not provide"
+                )
+            }
         }
     }
 }

crates/clawhdf5-format/src/extensible_array.rs

@@ -9,19 +9,23 @@ extern crate alloc;
 #[cfg(not(feature = "std"))]
 use alloc::{format, vec, vec::Vec};
 
+use crate::addr::to_usize;
 use crate::chunk_grid::ChunkGrid;
 use crate::chunked_read::ChunkInfo;
 use crate::error::FormatError;
+use crate::storage::{PAGED_BLOCK_ONE_READ_MAX, Storage, Window, read_exact_at};
 
 /// Verify the Jenkins lookup3 checksum stored immediately after
-/// `data[start..end]`, as every Extensible Array structure carries one.
+/// `data[start..end]`, as every Extensible Array structure carries one. `w`
+/// is a window of the file and `start`/`end` are relative to it.
 ///
 /// A corrupt chunk index yields addresses pointing at the wrong bytes, so a
 /// mismatch is an error: otherwise the damage surfaces as plausible data read
 /// from the wrong chunk.
 #[cfg(feature = "checksum")]
-fn verify_checksum(data: &[u8], start: usize, end: usize) -> Result<(), FormatError> {
+fn verify_checksum(w: &Window<'_>, start: usize, end: usize) -> Result<(), FormatError> {
-    ensure_len(data, end, 4)?;
+    w.ensure(end, 4)?;
+    let data: &[u8] = &w.bytes;
     let stored = u32::from_le_bytes([data[end], data[end + 1], data[end + 2], data[end + 3]]);
     let computed = crate::checksum::jenkins_lookup3(&data[start..end]);
     if computed != stored {
@@ -34,7 +38,7 @@ fn verify_checksum(data: &[u8], start: usize, end: usize) -> Result<(), FormatEr
 }
 
 #[cfg(not(feature = "checksum"))]
-fn verify_checksum(_data: &[u8], _start: usize, _end: usize) -> Result<(), FormatError> {
+fn verify_checksum(_w: &Window<'_>, _start: usize, _end: usize) -> Result<(), FormatError> {
     Ok(())
 }
 
@@ -80,19 +84,6 @@ fn read_offset(data: &[u8], pos: usize, size: u8) -> Result<u64, FormatError> {
     })
 }
 
-fn ensure_len(data: &[u8], offset: usize, needed: usize) -> Result<(), FormatError> {
-    if offset
-        .checked_add(needed)
-        .is_none_or(|end| end > data.len())
-    {
-        return Err(FormatError::UnexpectedEof {
-            expected: offset.saturating_add(needed),
-            available: data.len(),
-        });
-    }
-    Ok(())
-}
-
 fn is_undefined_addr(addr: u64, offset_size: u8) -> bool {
     match offset_size {
         2 => addr == 0xFFFF,
@@ -130,6 +121,16 @@ impl ExtensibleArrayHeader {
         offset: usize,
         offset_size: u8,
         length_size: u8,
+    ) -> Result<Self, FormatError> {
+        Self::parse_in(file_data, offset as u64, offset_size, length_size)
+    }
+
+    /// [`Self::parse`] over any [`Storage`]: one read of the header.
+    pub fn parse_in<S: Storage + ?Sized>(
+        file: &S,
+        offset: u64,
+        offset_size: u8,
+        length_size: u8,
     ) -> Result<Self, FormatError> {
         // EAHD: signature(4) + version(1) + client_id(1) + element_size(1) +
         //   max_nelmts_bits(1) + idx_blk_elmts(1) + min_dblk_nelmts(1) +
@@ -137,9 +138,10 @@ impl ExtensibleArrayHeader {
         //   6 stats fields (each length_size) + index_block_address(offset_size) + checksum(4)
         let min_size =
             4 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 6 * length_size as usize + offset_size as usize + 4;
-        ensure_len(file_data, offset, min_size)?;
+        let w = Window::read(file, offset, min_size)?;
+        w.ensure(0, min_size)?;
 
-        let d = &file_data[offset..];
+        let d: &[u8] = &w.bytes;
         if &d[0..4] != b"EAHD" {
             return Err(FormatError::ChunkedReadError(
                 "invalid Extensible Array header signature".into(),
@@ -172,7 +174,7 @@ impl ExtensibleArrayHeader {
         pos += ls; // skip max_idx_set (6th stats field)
         let index_block_address = read_offset(d, pos, offset_size)?;
         pos += offset_size as usize;
-        verify_checksum(file_data, offset, offset + pos)?;
+        verify_checksum(&w, 0, pos)?;
 
         Ok(ExtensibleArrayHeader {
             client_id,
@@ -193,11 +195,11 @@ impl ExtensibleArrayHeader {
     }
 }
 
-/// Read a single element from the extensible array element data.
+/// Read a single element at offset `pos` of the window `w`.
 /// Returns (chunk_info, bytes_consumed) or None if unallocated.
 #[allow(clippy::too_many_arguments)]
 fn read_element(
-    data: &[u8],
+    w: &Window<'_>,
     pos: usize,
     client_id: u8,
     element_size: u8,
@@ -207,15 +209,11 @@ fn read_element(
     grid: &ChunkGrid,
 ) -> Result<(Option<ChunkInfo>, usize), FormatError> {
     let os = offset_size as usize;
+    let data: &[u8] = &w.bytes;
 
     if client_id == 0 {
         // Non-filtered: just address
-        if pos + os > data.len() {
+        w.ensure(pos, os)?;
-            return Err(FormatError::UnexpectedEof {
-                expected: pos + os,
-                available: data.len(),
-            });
-        }
         if is_undefined(data, pos, offset_size) {
             return Ok((None, os));
         }
@@ -226,7 +224,7 @@ fn read_element(
         };
         Ok((
             Some(ChunkInfo {
-                chunk_size: chunk_byte_size as u32,
+                chunk_size: chunk_byte_size,
                 filter_mask: 0,
                 offsets,
                 address,
@@ -243,15 +241,7 @@ fn read_element(
         }
         let chunk_size_bytes = es - os - 4;
         let elem_total = os + chunk_size_bytes + 4;
-        if pos
+        w.ensure(pos, elem_total)?;
-            .checked_add(elem_total)
-            .is_none_or(|end| end > data.len())
-        {
-            return Err(FormatError::UnexpectedEof {
-                expected: pos.saturating_add(elem_total),
-                available: data.len(),
-            });
-        }
         if is_undefined(data, pos, offset_size) {
             return Ok((None, elem_total));
         }
@@ -269,7 +259,7 @@ fn read_element(
         };
         Ok((
             Some(ChunkInfo {
-                chunk_size: chunk_size as u32,
+                chunk_size,
                 filter_mask,
                 offsets,
                 address,
@@ -315,9 +305,9 @@ fn page_nelmts(header: &ExtensibleArrayHeader) -> Option<usize> {
 /// paged. The bitmap lives in the super block, not here — a paged data block
 /// stores only its prefix, then one slot per page.
 #[allow(clippy::too_many_arguments)]
-fn read_data_block_elements(
+fn read_data_block_elements<S: Storage + ?Sized>(
-    file_data: &[u8],
+    file: &S,
-    db_offset: usize,
+    db_offset: u64,
     nelmts: usize,
     header: &ExtensibleArrayHeader,
     offset_size: u8,
@@ -330,21 +320,28 @@ fn read_data_block_elements(
     // EADB: signature(4) + version(1) + client_id(1) + header_address(offset_size)
     //       + block offset(arr_off_size)
     let db_header_size = 4 + 1 + 1 + offset_size as usize + arr_off_size(header);
-    ensure_len(file_data, db_offset, db_header_size)?;
+    let prefix = read_exact_at(file, db_offset, db_header_size)?;
 
-    if &file_data[db_offset..db_offset + 4] != b"EADB" {
+    if &prefix[0..4] != b"EADB" {
         return Err(FormatError::ChunkedReadError(
             "invalid Extensible Array data block signature".into(),
         ));
     }
 
-    let mut pos = db_offset + db_header_size;
+    // Positions below are relative to the data block.
+    let mut pos = db_header_size;
     let page = page_nelmts(header).ok_or_else(|| {
         FormatError::Overflow("Extensible Array page element count overflows usize".into())
     })?;
+    let elem_bytes = if header.client_id == 0 {
+        offset_size as usize
+    } else {
+        header.element_size as usize
+    };
 
     let mut chunks = Vec::new();
-    let read_run = |from: usize,
+    let read_run = |w: &Window<'_>,
+                    from: usize,
                     count: usize,
                     first_index: usize,
                     chunks: &mut Vec<ChunkInfo>|
@@ -352,7 +349,7 @@ fn read_data_block_elements(
         let mut p = from;
         for i in 0..count {
             let (info, consumed) = read_element(
-                file_data,
+                w,
                 p,
                 header.client_id,
                 header.element_size,
@@ -370,18 +367,19 @@ fn read_data_block_elements(
     };
 
     if nelmts <= page {
-        // Prefix and elements are covered by one checksum.
+        // Prefix and elements are covered by one checksum. One window holds
-        let elem_bytes = if header.client_id == 0 {
+        // all of it (or ends at the end of the file), so its bounds checks
-            offset_size as usize
+        // are the whole-file ones.
-        } else {
-            header.element_size as usize
-        };
         let end = nelmts
             .checked_mul(elem_bytes)
             .and_then(|b| pos.checked_add(b))
             .ok_or_else(|| FormatError::Overflow("Extensible Array data block span".into()))?;
-        verify_checksum(file_data, db_offset, end)?;
+        // The checksum's bounds check comes first: make it before reading.
-        read_run(pos, nelmts, start_index, &mut chunks)?;
+        #[cfg(feature = "checksum")]
+        Window::check_extent(file, db_offset, end, 4)?;
+        let w = Window::read(file, db_offset, end.saturating_add(4))?;
+        verify_checksum(&w, 0, end)?;
+        read_run(&w, pos, nelmts, start_index, &mut chunks)?;
         return Ok(chunks);
     }
 
@@ -389,18 +387,32 @@ fn read_data_block_elements(
     // each holding `page` elements followed by a checksum. Pages whose bit is
     // clear were never written; their slot still occupies the file, so stride
     // over it rather than reading zeros as addresses.
-    verify_checksum(file_data, db_offset, pos)?;
+    let npages = nelmts.div_ceil(page);
-    pos += 4;
+    // The whole data block in one window when it is small: every position
-    let elem_bytes = if header.client_id == 0 {
+    // checked below lies inside it (or past the end of the file). A larger
-        offset_size as usize
+    // block is read as its prefix, then each page in use on its own.
+    let block_len = pos
+        .saturating_add(4)
+        .saturating_add(npages.saturating_mul(page.saturating_mul(elem_bytes).saturating_add(4)));
+    let whole = if block_len <= PAGED_BLOCK_ONE_READ_MAX {
+        Some(Window::read(file, db_offset, block_len)?)
     } else {
-        header.element_size as usize
+        None
     };
+    let head_w;
+    let head = match &whole {
+        Some(w) => w,
+        None => {
+            head_w = Window::read(file, db_offset, pos + 4)?;
+            &head_w
+        }
+    };
+    verify_checksum(head, 0, pos)?;
+    pos += 4;
     let page_stride = page
         .checked_mul(elem_bytes)
         .and_then(|b| b.checked_add(4))
         .ok_or_else(|| FormatError::Overflow("Extensible Array page stride".into()))?;
-    let npages = nelmts.div_ceil(page);
     for p in 0..npages {
         // One bit per page across the whole super block, packed contiguously
         // and MSB-first within each byte, as H5VM_bit_get reads it.
@@ -410,10 +422,20 @@ fn read_data_block_elements(
             .is_some_and(|byte| byte & (0x80 >> (bit % 8)) != 0);
         if initialised {
             let count = core::cmp::min(page, nelmts - p * page);
+            // `w` holds the page from `base` on (positions below are
+            // relative to it, and `pos` to the data block).
+            let page_w;
+            let (w, base) = match &whole {
+                Some(w) => (w, 0),
+                None => {
+                    page_w = Window::read(file, db_offset.saturating_add(pos as u64), page_stride)?;
+                    (&page_w, pos)
+                }
+            };
             // Each page carries its own checksum, over a full page's worth of
             // slots even when the last one holds fewer live elements.
-            verify_checksum(file_data, pos, pos + page * elem_bytes)?;
+            verify_checksum(w, pos - base, pos - base + page * elem_bytes)?;
-            read_run(pos, count, start_index + p * page, &mut chunks)?;
+            read_run(w, pos - base, count, start_index + p * page, &mut chunks)?;
         }
         pos = pos
             .checked_add(page_stride)
@@ -435,6 +457,32 @@ pub fn read_extensible_array_chunks(
     chunk_dimensions: &[u32],
     element_size: u32,
     offset_size: u8,
+    length_size: u8,
+) -> Result<Vec<ChunkInfo>, FormatError> {
+    read_extensible_array_chunks_in(
+        &file_data,
+        header,
+        dataset_dims,
+        max_dims,
+        chunk_dimensions,
+        element_size,
+        offset_size,
+        length_size,
+    )
+}
+
+/// [`read_extensible_array_chunks`] over any [`Storage`]: one read of the
+/// index block's prefix, one of the whole index block, and the same for
+/// every super block and data block it references.
+#[allow(clippy::too_many_arguments)]
+pub fn read_extensible_array_chunks_in<S: Storage + ?Sized>(
+    file: &S,
+    header: &ExtensibleArrayHeader,
+    dataset_dims: &[u64],
+    max_dims: Option<&[u64]>,
+    chunk_dimensions: &[u32],
+    element_size: u32,
+    offset_size: u8,
     _length_size: u8,
 ) -> Result<Vec<ChunkInfo>, FormatError> {
     let os = offset_size as usize;
@@ -451,19 +499,20 @@ pub fn read_extensible_array_chunks(
     // Parse index block (EAIB): signature(4) + version(1) + client_id(1)
     // + header address(offset_size), then the inline elements, then the
     // direct data block addresses, then the super block addresses.
-    let ib_offset = header.index_block_address as usize;
+    // Positions below are relative to the index block.
+    let ib_offset = header.index_block_address;
     let ib_header_size = 4 + 1 + 1 + os;
-    ensure_len(file_data, ib_offset, ib_header_size)?;
+    let prefix = read_exact_at(file, ib_offset, ib_header_size)?;
 
-    if &file_data[ib_offset..ib_offset + 4] != b"EAIB" {
+    if &prefix[0..4] != b"EAIB" {
         return Err(FormatError::ChunkedReadError(
             "invalid Extensible Array index block signature".into(),
         ));
     }
-    let mut pos = ib_offset + ib_header_size;
+    let mut pos = ib_header_size;
 
     let mut chunks = Vec::new();
-    let total_elements = header.num_elements as usize;
+    let total_elements = to_usize(header.num_elements)?;
 
     let dmin = header.min_dblk_nelmts as usize;
     if dmin == 0 || !dmin.is_power_of_two() {
@@ -520,13 +569,19 @@ pub fn read_extensible_array_chunks(
                 .and_then(|n| n.checked_mul(os).and_then(|b| p.checked_add(b)))
         })
         .ok_or_else(|| FormatError::Overflow("Extensible Array index block span".into()))?;
-    verify_checksum(file_data, ib_offset, ib_end)?;
+    // The whole index block in one window: every position read below is
+    // before `ib_end`.
+    // The checksum's bounds check comes first: make it before reading.
+    #[cfg(feature = "checksum")]
+    Window::check_extent(file, ib_offset, ib_end, 4)?;
+    let w = Window::read(file, ib_offset, ib_end.saturating_add(4))?;
+    verify_checksum(&w, 0, ib_end)?;
 
     // 1. Elements stored inline in the index block.
     let n_inline = (header.idx_blk_elmts as usize).min(total_elements);
     for i in 0..n_inline {
         let (info, consumed) = read_element(
-            file_data,
+            &w,
             pos,
             header.client_id,
             header.element_size,
@@ -550,8 +605,8 @@ pub fn read_extensible_array_chunks(
         if global_index >= total_elements {
             return Ok(chunks);
         }
-        ensure_len(file_data, pos, os)?;
+        w.ensure(pos, os)?;
-        let addr = read_offset(file_data, pos, offset_size)?;
+        let addr = read_offset(&w.bytes, pos, offset_size)?;
         pos += os;
         if !is_undefined_addr(addr, offset_size) {
             if dblk_nelmts > page_nelmts(header).unwrap_or(usize::MAX) {
@@ -562,8 +617,8 @@ pub fn read_extensible_array_chunks(
                 ));
             }
             chunks.extend(read_data_block_elements(
-                file_data,
+                file,
-                addr as usize,
+                addr,
                 dblk_nelmts,
                 header,
                 offset_size,
@@ -583,16 +638,16 @@ pub fn read_extensible_array_chunks(
         if global_index >= total_elements {
             break;
         }
-        ensure_len(file_data, pos, os)?;
+        w.ensure(pos, os)?;
-        let sb_addr = read_offset(file_data, pos, offset_size)?;
+        let sb_addr = read_offset(&w.bytes, pos, offset_size)?;
         pos += os;
         let (ndblks, dblk_nelmts) = sblk_info(u, dmin).ok_or_else(|| {
             FormatError::Overflow("Extensible Array super block layout overflows usize".into())
         })?;
         if !is_undefined_addr(sb_addr, offset_size) {
             chunks.extend(read_super_block(
-                file_data,
+                file,
-                sb_addr as usize,
+                sb_addr,
                 ndblks,
                 dblk_nelmts,
                 header,
@@ -617,9 +672,9 @@ pub fn read_extensible_array_chunks(
 /// + block offset + the page-init bitmap for every data block it owns
 /// + one address per data block + checksum.
 #[allow(clippy::too_many_arguments)]
-fn read_super_block(
+fn read_super_block<S: Storage + ?Sized>(
-    file_data: &[u8],
+    file: &S,
-    sb_offset: usize,
+    sb_offset: u64,
     ndblks: usize,
     dblk_nelmts: usize,
     header: &ExtensibleArrayHeader,
@@ -630,9 +685,9 @@ fn read_super_block(
 ) -> Result<Vec<ChunkInfo>, FormatError> {
     let os = offset_size as usize;
     let sb_header_size = 4 + 1 + 1 + os + arr_off_size(header);
-    ensure_len(file_data, sb_offset, sb_header_size)?;
+    let prefix = read_exact_at(file, sb_offset, sb_header_size)?;
 
-    if &file_data[sb_offset..sb_offset + 4] != b"EASB" {
+    if &prefix[0..4] != b"EASB" {
         return Err(FormatError::ChunkedReadError(
             "invalid Extensible Array super block signature".into(),
         ));
@@ -654,29 +709,38 @@ fn read_super_block(
     let bitmap_bytes = per_dblk_bitmap
         .checked_mul(ndblks)
         .ok_or_else(|| FormatError::Overflow("Extensible Array page bitmap size".into()))?;
-    let bitmap_start = sb_offset + sb_header_size;
+    // Positions below are relative to the super block, whose bytes (up to
-    ensure_len(file_data, bitmap_start, bitmap_bytes)?;
+    // its checksum) are all in one window.
-    let bitmap = &file_data[bitmap_start..bitmap_start + bitmap_bytes];
+    let bitmap_start = sb_header_size;
-
+    // The bitmap's bounds check, then (with checksums) the checksum's, come
+    // before anything else is read from the block: make them before reading
+    // it, so size fields stretching it past the end of the file cost no read.
+    Window::check_extent(file, sb_offset, bitmap_start, bitmap_bytes)?;
     let mut pos = bitmap_start + bitmap_bytes;
-    let mut chunks = Vec::new();
-    let mut global_idx = start_index;
 
     // One checksum covers the prefix, the bitmap and every data block address.
     let sb_end = ndblks
         .checked_mul(os)
         .and_then(|b| pos.checked_add(b))
         .ok_or_else(|| FormatError::Overflow("Extensible Array super block span".into()))?;
-    verify_checksum(file_data, sb_offset, sb_end)?;
+    #[cfg(feature = "checksum")]
+    Window::check_extent(file, sb_offset, sb_end, 4)?;
+    let w = Window::read(file, sb_offset, sb_end.saturating_add(4))?;
+    w.ensure(bitmap_start, bitmap_bytes)?;
+    let bitmap = &w.bytes[bitmap_start..bitmap_start + bitmap_bytes];
+
+    let mut chunks = Vec::new();
+    let mut global_idx = start_index;
+    verify_checksum(&w, 0, sb_end)?;
 
     for i in 0..ndblks {
-        ensure_len(file_data, pos, os)?;
+        w.ensure(pos, os)?;
-        let addr = read_offset(file_data, pos, offset_size)?;
+        let addr = read_offset(&w.bytes, pos, offset_size)?;
         pos += os;
         if !is_undefined_addr(addr, offset_size) {
             chunks.extend(read_data_block_elements(
-                file_data,
+                file,
-                addr as usize,
+                addr,
                 dblk_nelmts,
                 header,
                 offset_size,
@@ -882,16 +946,16 @@ mod tests {
         assert_eq!(chunks.len(), 2);
         assert_eq!(chunks[0].address, base_addr);
         assert_eq!(chunks[0].offsets, vec![0]);
-        assert_eq!(chunks[0].chunk_size, chunk_byte_size as u32);
+        assert_eq!(chunks[0].chunk_size, chunk_byte_size);
         assert_eq!(chunks[1].address, base_addr + chunk_byte_size);
         assert_eq!(chunks[1].offsets, vec![20]);
     }
 
-    /// Build a synthetic EA with inline elements + one direct data block.
+    /// A synthetic EA with inline elements + one direct data block: the
-    #[test]
+    /// file, with the header at 0x100 (8-byte offsets and lengths, 4 chunks
-    fn read_inline_plus_data_blocks() {
+    /// of 10 elements from 0x1000 on).
+    fn build_inline_plus_data_blocks() -> Vec<u8> {
         let os: u8 = 8;
-        let ls: u8 = 8;
         let osv = os as usize;
         let chunk_byte_size = 10u64 * 8; // 10 elements × 8 bytes
         let idx_blk_elmts = 2u8;
@@ -981,8 +1045,17 @@ mod tests {
             dbpos += osv;
         }
         stamp_checksum(&mut file_data, aedb_offset, dbpos);
+        file_data
+    }
 
-        let header = ExtensibleArrayHeader::parse(&file_data, aehd_offset, os, ls).unwrap();
+    /// Build a synthetic EA with inline elements + one direct data block.
+    #[test]
+    fn read_inline_plus_data_blocks() {
+        let (os, ls) = (8u8, 8u8);
+        let chunk_byte_size = 10u64 * 8;
+        let base_addr = 0x1000u64;
+        let file_data = build_inline_plus_data_blocks();
+        let header = ExtensibleArrayHeader::parse(&file_data, 0x100, os, ls).unwrap();
         let ds_dims = vec![40u64];
         let chunk_dims = vec![10u32];
         let chunks = read_extensible_array_chunks(
@@ -1018,7 +1091,8 @@ mod tests {
     fn read_element_unallocated() {
         let data = vec![0xFFu8; 16];
         let grid = ChunkGrid::fixed_array(&[50], None, &[10]).unwrap();
-        let (info, consumed) = read_element(&data, 0, 0, 8, 8, 80, 0, &grid).unwrap();
+        let (info, consumed) =
+            read_element(&Window::whole(&data), 0, 0, 8, 8, 80, 0, &grid).unwrap();
         assert!(info.is_none());
         assert_eq!(consumed, 8);
     }
@@ -1038,8 +1112,17 @@ mod tests {
         data[12..16].copy_from_slice(&0u32.to_le_bytes());
 
         let grid = ChunkGrid::fixed_array(&[50], None, &[10]).unwrap();
-        let (info, consumed) =
+        let (info, consumed) = read_element(
-            read_element(&data, 0, 1, elem_size as u8, os, 80, 2, &grid).unwrap();
+            &Window::whole(&data),
+            0,
+            1,
+            elem_size as u8,
+            os,
+            80,
+            2,
+            &grid,
+        )
+        .unwrap();
         let ci = info.unwrap();
         assert_eq!(ci.address, 0x2000);
         assert_eq!(ci.chunk_size, 120);
@@ -1047,4 +1130,38 @@ mod tests {
         assert_eq!(ci.offsets, vec![20]);
         assert_eq!(consumed, elem_size);
     }
+
+    /// The Storage path reads exactly what the slice path reads: the array
+    /// whole, cut at every length through its structures, and with a byte
+    /// damaged in each of them, through a read_at-only CountingStorage.
+    #[test]
+    fn storage_reads_match_slice_reads() {
+        use crate::storage::CountingStorage;
+        let full = build_inline_plus_data_blocks();
+        let mut files = Vec::new();
+        for cut in 0x100..0x340 {
+            files.push(full[..cut].to_vec());
+        }
+        for at in [0x104, 0x150, 0x204, 0x216, 0x230, 0x304, 0x318] {
+            let mut damaged = full.clone();
+            damaged[at] ^= 1;
+            files.push(damaged);
+        }
+        files.push(full);
+        let mut compared = 0;
+        for f in files {
+            let storage = CountingStorage::new(f.clone());
+            let want = ExtensibleArrayHeader::parse(&f, 0x100, 8, 8);
+            let got = ExtensibleArrayHeader::parse_in(&storage, 0x100, 8, 8);
+            assert_eq!(format!("{got:?}"), format!("{want:?}"));
+            let Ok(h) = want else { continue };
+            for dims in [&[40u64][..], &[25]] {
+                let want = read_extensible_array_chunks(&f, &h, dims, None, &[10], 8, 8, 8);
+                let got = read_extensible_array_chunks_in(&storage, &h, dims, None, &[10], 8, 8, 8);
+                assert_eq!(format!("{got:?}"), format!("{want:?}"), "{} bytes", f.len());
+                compared += 1;
+            }
+        }
+        assert!(compared > 100);
+    }
 }

crates/clawhdf5-format/src/fill_value.rs

@@ -12,7 +12,8 @@
 #[cfg(not(feature = "std"))]
 use alloc::{format, vec, vec::Vec};
 
-use crate::chunked_read::{alloc_output, checked_byte_len, list_chunks};
+use crate::addr::to_usize;
+use crate::chunked_read::{alloc_output, checked_byte_len, list_chunks_in};
 use crate::data_layout::DataLayout;
 use crate::dataspace::Dataspace;
 use crate::error::FormatError;
@@ -116,9 +117,22 @@ pub fn dataset_fill_value_in(
     messages: &[HeaderMessage],
     offset_size: u8,
     length_size: u8,
+) -> Result<Option<Vec<u8>>, FormatError> {
+    dataset_fill_value_from_storage(&file_data, messages, offset_size, length_size)
+}
+
+/// [`dataset_fill_value_in`] with the file behind any
+/// [`Storage`](crate::storage::Storage) (a `&dyn Storage` too). (The trait
+/// is not imported here: its `len` would shadow the slice method in this
+/// module.)
+pub fn dataset_fill_value_from_storage<S: crate::storage::Storage + ?Sized>(
+    file: &S,
+    messages: &[HeaderMessage],
+    offset_size: u8,
+    length_size: u8,
 ) -> Result<Option<Vec<u8>>, FormatError> {
     fill_value_from(messages, |msg| {
-        crate::shared_message::message_data_with_sohm(file_data, msg, offset_size, length_size)
+        crate::shared_message::message_data_with_sohm_in(file, msg, offset_size, length_size)
             .map(|data| data.into_owned())
     })
 }
@@ -199,6 +213,30 @@ pub fn read_full_with_fill<E: From<FormatError>>(
     offset_size: u8,
     length_size: u8,
     read: impl FnOnce() -> Result<Vec<u8>, E>,
+) -> Result<Vec<u8>, E> {
+    read_full_with_fill_in(
+        messages,
+        file_data,
+        layout,
+        dataspace,
+        elem_size,
+        offset_size,
+        length_size,
+        read,
+    )
+}
+
+/// [`read_full_with_fill`] over any [`Storage`](crate::storage::Storage).
+#[allow(clippy::too_many_arguments)]
+pub fn read_full_with_fill_in<E: From<FormatError>, S: crate::storage::Storage + ?Sized>(
+    messages: &[HeaderMessage],
+    file_data: &S,
+    layout: &DataLayout,
+    dataspace: &Dataspace,
+    elem_size: usize,
+    offset_size: u8,
+    length_size: u8,
+    read: impl FnOnce() -> Result<Vec<u8>, E>,
 ) -> Result<Vec<u8>, E> {
     // A dataset with external raw data also has no data address in this
     // file. It is NOT unallocated — its values live elsewhere — so it must
@@ -209,12 +247,12 @@ pub fn read_full_with_fill<E: From<FormatError>>(
     {
         return Err(FormatError::ExternalDataFilesUnsupported.into());
     }
-    let fill = dataset_fill_value_in(file_data, messages, offset_size, length_size)?;
+    let fill = dataset_fill_value_from_storage(file_data, messages, offset_size, length_size)?;
     if !has_storage(layout) {
         return Ok(filled_dataset(dataspace, elem_size, fill.as_deref())?);
     }
     let mut output = read()?;
-    apply_to_unallocated_chunks(
+    apply_to_unallocated_chunks_in(
         &mut output,
         file_data,
         layout,
@@ -240,6 +278,30 @@ pub fn apply_to_unallocated_chunks(
     fill: Option<&[u8]>,
     offset_size: u8,
     length_size: u8,
+) -> Result<(), FormatError> {
+    apply_to_unallocated_chunks_in(
+        output,
+        file_data,
+        layout,
+        dataspace,
+        elem_size,
+        fill,
+        offset_size,
+        length_size,
+    )
+}
+
+/// [`apply_to_unallocated_chunks`] over any [`Storage`](crate::storage::Storage).
+#[allow(clippy::too_many_arguments)]
+pub fn apply_to_unallocated_chunks_in<S: crate::storage::Storage + ?Sized>(
+    output: &mut [u8],
+    file_data: &S,
+    layout: &DataLayout,
+    dataspace: &Dataspace,
+    elem_size: usize,
+    fill: Option<&[u8]>,
+    offset_size: u8,
+    length_size: u8,
 ) -> Result<(), FormatError> {
     let Some(fill) = fill.filter(|f| f.len() == elem_size && !is_default(Some(f))) else {
         return Ok(());
@@ -247,7 +309,7 @@ pub fn apply_to_unallocated_chunks(
     if !matches!(layout, DataLayout::Chunked { .. }) || elem_size == 0 {
         return Ok(());
     }
-    let (chunks, chunk_dims) = list_chunks(
+    let (chunks, chunk_dims) = list_chunks_in(
         file_data,
         layout,
         dataspace,
@@ -256,7 +318,11 @@ pub fn apply_to_unallocated_chunks(
         length_size,
     )?;
     let rank = chunk_dims.len();
-    let ds_dims: Vec<usize> = dataspace.dimensions.iter().map(|&d| d as usize).collect();
+    let ds_dims: Vec<usize> = dataspace
+        .dimensions
+        .iter()
+        .map(|&d| to_usize(d))
+        .collect::<Result<_, _>>()?;
     if rank == 0 || ds_dims.len() != rank || chunk_dims.contains(&0) {
         return Ok(());
     }
@@ -288,7 +354,7 @@ pub fn apply_to_unallocated_chunks(
         let mut cell = 0usize;
         let mut in_range = true;
         for d in 0..rank {
-            let coord = chunk.offsets[d] as usize / chunk_dims[d];
+            let coord = to_usize(chunk.offsets[d])? / chunk_dims[d];
             if coord >= grid[d] {
                 in_range = false;
                 break;
@@ -439,4 +505,37 @@ mod tests {
             .collect();
         assert_eq!(filled, [2, 3, 7, 8]);
     }
+
+    /// Fill values, shared ones in the SOHM heap included, resolve
+    /// identically through a read_at-only CountingStorage.
+    #[test]
+    fn storage_reads_match_slice_reads() {
+        use crate::object_header::ObjectHeader;
+        use crate::storage::CountingStorage;
+        let file: &[u8] = include_bytes!("../tests/fixtures/shared_fill_value.h5");
+        let sb = crate::superblock::Superblock::parse(file, 0).unwrap();
+        let (os, ls) = (sb.offset_size, sb.length_size);
+        let storage = CountingStorage::new(file.to_vec());
+        let mut shared = 0;
+        let children =
+            crate::group_v2::resolve_group_children(file, &sb, sb.root_group_address).unwrap();
+        assert!(children.len() >= 3);
+        for child in children {
+            let h =
+                ObjectHeader::parse(file, child.object_header_address as usize, os, ls).unwrap();
+            shared += h
+                .messages
+                .iter()
+                .filter(|m| {
+                    m.msg_type == MessageType::FillValue
+                        && crate::shared_message::is_shared(m.flags)
+                })
+                .count();
+            let want = dataset_fill_value_in(file, &h.messages, os, ls);
+            assert_eq!(want, Ok(Some((-7i32).to_le_bytes().to_vec())));
+            let got = dataset_fill_value_from_storage(&storage, &h.messages, os, ls);
+            assert_eq!(got, want, "{}", child.name);
+        }
+        assert!(shared >= 2);
+    }
 }

crates/clawhdf5-format/src/filter_pipeline.rs

@@ -27,7 +27,8 @@ pub const FILTER_LZF: u16 = 32000;
 pub const FILTER_BLOSC: u16 = 32001;
 /// Bitshuffle, optionally with LZ4 or Zstandard (hdf5plugin's `Bitshuffle`).
 pub const FILTER_BITSHUFFLE: u16 = 32008;
-/// ZFP lossy floating-point compression (hdf5plugin's `Zfp`). Not supported.
+/// ZFP lossy (and lossless) compression of numeric arrays (H5Z-ZFP;
+/// hdf5plugin's `Zfp`). Read-only, with the `zfp` feature.
 pub const FILTER_ZFP: u16 = 32013;
 /// Blosc 2 (hdf5plugin's `Blosc2`).
 pub const FILTER_BLOSC2: u16 = 32026;

crates/clawhdf5-format/src/filter_registry.rs

@@ -5,7 +5,8 @@
 //! * **Built-in filters** — a static table of the filters compiled into this
 //!   build: the HDF5 standard filters (deflate, shuffle, Fletcher32, szip,
 //!   N-Bit, scale-offset) and the plugin filters whose cargo features are
-//!   enabled (LZ4, Zstandard, pcodec, LZF, bitshuffle, bzip2, blosc).
+//!   enabled (LZ4, Zstandard, pcodec, LZF, bitshuffle, bzip2, blosc,
+//!   blosc2, zfp).
 //!   [`builtin_filters`] lists them.
 //! * **Registered filters** (`std` only) — codecs the application supplies
 //!   for any other ID with [`register_filter`] (a [`FilterCodec`], or just a
@@ -161,12 +162,12 @@ pub fn known_filter(id: u16) -> Option<(&'static str, Option<&'static str>)> {
         32001 => ("Blosc", Some("blosc")),
         32004 => ("LZ4", Some("lz4")),
         32008 => ("bitshuffle", Some("bitshuffle")),
-        32013 => ("ZFP", None),
+        32013 => ("ZFP", Some("zfp")),
         32015 => ("Zstandard", Some("zstd")),
         32019 => ("JPEG", None),
         32022 => ("BitGroom", None),
         32023 => ("Granular BitRound", None),
-        32026 => ("Blosc2", None),
+        32026 => ("Blosc2", Some("blosc2")),
         _ => return None,
     })
 }
@@ -187,6 +188,22 @@ pub fn is_filter_available(id: u16) -> bool {
     }
 }
 
+/// Whether chunks filtered with `id` may be decoded by a codec the
+/// application registered (whose stored sizes this crate cannot bound).
+pub(crate) fn may_be_registered(id: u16) -> bool {
+    if builtin_filter(id).is_some_and(|b| !b.is_shared()) {
+        return false;
+    }
+    #[cfg(feature = "std")]
+    {
+        registered(id).is_some()
+    }
+    #[cfg(not(feature = "std"))]
+    {
+        false
+    }
+}
+
 #[cfg(feature = "std")]
 mod custom {
     use super::FilterCodec;
@@ -452,12 +469,14 @@ pub(crate) mod tests {
     #[test]
     fn unsupported_filter_error_names_the_filter() {
         let msg = FormatError::UnsupportedFilter(32026).to_string();
+        assert!(msg.contains("Blosc2") && msg.contains("`blosc2`"), "{msg}");
+        let msg = FormatError::UnsupportedFilter(32013).to_string();
+        assert!(msg.contains("ZFP") && msg.contains("`zfp`"), "{msg}");
+        let msg = FormatError::UnsupportedFilter(32019).to_string();
         assert!(
-            msg.contains("Blosc2") && msg.contains("not implemented"),
+            msg.contains("JPEG") && msg.contains("not implemented"),
             "{msg}"
         );
-        let msg = FormatError::UnsupportedFilter(32013).to_string();
-        assert!(msg.contains("ZFP"), "{msg}");
         let msg = FormatError::UnsupportedFilter(32000).to_string();
         assert!(msg.contains("LZF") && msg.contains("`lzf`"), "{msg}");
         assert_eq!(

crates/clawhdf5-format/src/filters_bitshuffle.rs

@@ -218,12 +218,20 @@ pub(crate) fn bitshuffle_decode(
 }
 
 /// Decode Zstandard frames into exactly `dst`, failing if they hold more.
+///
+/// ruzstd reserves a frame's declared window (by default up to 100 MiB)
+/// before decoding it, so the window is capped at what the output could
+/// need: twice `dst` (window sizes are rounded up), and at least 128 KiB.
+/// The encoders behind these filters (c-blosc, c-blosc2, bitshuffle)
+/// compress each block in one call with its size known, so libzstd's
+/// window never exceeds the block.
 #[cfg(any(feature = "bitshuffle", feature = "blosc"))]
 pub(crate) fn zstd_decode_into(
     decoder: &mut ruzstd::decoding::FrameDecoder,
     frames: &[u8],
     dst: &mut [u8],
 ) -> Result<usize, FormatError> {
+    decoder.set_max_window_size((2 * dst.len()).max(1 << 17) as u64);
     decoder
         .decode_all(frames, dst)
         .map_err(|e| FormatError::DecompressionError(format!("zstd: {e}")))

crates/clawhdf5-format/src/filters_blosc.rs

@@ -49,7 +49,7 @@ fn le32(b: &[u8], at: usize) -> Result<usize, FormatError> {
 
 /// The codec inside a Blosc frame (flags bits 5-7).
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
-enum Codec {
+pub(crate) enum Codec {
     BloscLz,
     Lz4,
     Snappy,
@@ -58,7 +58,7 @@ enum Codec {
 }
 
 impl Codec {
-    fn from_flags(flags: u8) -> Result<Codec, FormatError> {
+    pub(crate) fn from_flags(flags: u8) -> Result<Codec, FormatError> {
         match flags >> 5 {
             0 => Ok(Codec::BloscLz),
             1 => Ok(Codec::Lz4),
@@ -71,7 +71,7 @@ impl Codec {
 }
 
 /// Decode one codec stream into exactly `dst`.
-fn decode_stream(
+pub(crate) fn decode_stream(
     codec: Codec,
     src: &[u8],
     dst: &mut [u8],

crates/clawhdf5-format/src/filters_bzip2.rs

@@ -4,6 +4,7 @@
 //! the compression level). Decoded with the `bzip2` crate's default backend,
 //! `libbz2-rs-sys`, a pure-Rust port of libbzip2.
 
+use crate::addr::saturating_usize;
 use crate::error::FormatError;
 use crate::filter_registry::FilterContext;
 
@@ -28,7 +29,7 @@ pub(crate) fn bzip2_decode(input: &[u8], ctx: &FilterContext<'_>) -> Result<Vec<
     loop {
         let (in_before, out_before) = (dec.total_in(), dec.total_out());
         let status = dec
-            .decompress_vec(&input[in_before as usize..], &mut out)
+            .decompress_vec(&input[saturating_usize(in_before)..], &mut out)
             .map_err(|e| err(&e.to_string()))?;
         if out.len() > limit {
             return Err(err("output exceeds the chunk size"));
@@ -43,7 +44,7 @@ pub(crate) fn bzip2_decode(input: &[u8], ctx: &FilterContext<'_>) -> Result<Vec<
                 .max(1);
             out.try_reserve_exact(grow)
                 .map_err(|_| err("cannot allocate the output buffer"))?;
-        } else if dec.total_in() as usize >= input.len()
+        } else if saturating_usize(dec.total_in()) >= input.len()
             || (dec.total_in(), dec.total_out()) == (in_before, out_before)
         {
             return Err(err("truncated stream"));
@@ -61,7 +62,7 @@ pub(crate) fn bzip2_encode(input: &[u8], ctx: &FilterContext<'_>) -> Result<Vec<
     // bzip2's worst case is about 1% + 600 bytes over the input.
     let mut out = Vec::with_capacity(input.len() + input.len() / 100 + 600);
     loop {
-        let consumed = enc.total_in() as usize;
+        let consumed = saturating_usize(enc.total_in());
         let status = enc
             .compress_vec(&input[consumed..], &mut out, Action::Finish)
             .map_err(|e| cerr(e.to_string()))?;

crates/clawhdf5-format/src/fixed_array.rs

@@ -6,19 +6,23 @@ extern crate alloc;
 #[cfg(not(feature = "std"))]
 use alloc::{format, vec, vec::Vec};
 
+use crate::addr::to_usize;
 use crate::chunk_grid::ChunkGrid;
 use crate::chunked_read::ChunkInfo;
 use crate::error::FormatError;
+use crate::storage::{PAGED_BLOCK_ONE_READ_MAX, Storage, Window, len_usize, read_exact_at};
 
 /// Verify the Jenkins lookup3 checksum stored immediately after
-/// `data[start..end]`, as every Fixed Array structure carries one.
+/// `data[start..end]`, as every Fixed Array structure carries one. `w` is
+/// a window of the file and `start`/`end` are relative to it.
 ///
 /// A corrupt chunk index silently yields addresses pointing at the wrong
 /// bytes, so a mismatch has to be an error rather than a shrug: without this
 /// the damage surfaces as plausible-looking data from the wrong chunk.
 #[cfg(feature = "checksum")]
-fn verify_checksum(data: &[u8], start: usize, end: usize) -> Result<(), FormatError> {
+fn verify_checksum(w: &Window<'_>, start: usize, end: usize) -> Result<(), FormatError> {
-    ensure_len(data, end, 4)?;
+    w.ensure(end, 4)?;
+    let data = &w.bytes;
     let stored = u32::from_le_bytes([data[end], data[end + 1], data[end + 2], data[end + 3]]);
     let computed = crate::checksum::jenkins_lookup3(&data[start..end]);
     if computed != stored {
@@ -31,7 +35,7 @@ fn verify_checksum(data: &[u8], start: usize, end: usize) -> Result<(), FormatEr
 }
 
 #[cfg(not(feature = "checksum"))]
-fn verify_checksum(_data: &[u8], _start: usize, _end: usize) -> Result<(), FormatError> {
+fn verify_checksum(_w: &Window<'_>, _start: usize, _end: usize) -> Result<(), FormatError> {
     Ok(())
 }
 
@@ -73,19 +77,6 @@ fn read_length(data: &[u8], pos: usize, size: u8) -> Result<u64, FormatError> {
     read_offset(data, pos, size)
 }
 
-fn ensure_len(data: &[u8], offset: usize, needed: usize) -> Result<(), FormatError> {
-    if offset
-        .checked_add(needed)
-        .is_none_or(|end| end > data.len())
-    {
-        return Err(FormatError::UnexpectedEof {
-            expected: offset.saturating_add(needed),
-            available: data.len(),
-        });
-    }
-    Ok(())
-}
-
 fn is_undefined(data: &[u8], pos: usize, size: u8) -> bool {
     let s = size as usize;
     if pos + s > data.len() {
@@ -101,13 +92,24 @@ impl FixedArrayHeader {
         offset: usize,
         offset_size: u8,
         length_size: u8,
+    ) -> Result<Self, FormatError> {
+        Self::parse_in(file_data, offset as u64, offset_size, length_size)
+    }
+
+    /// [`Self::parse`] over any [`Storage`]: one read of the header.
+    pub fn parse_in<S: Storage + ?Sized>(
+        file: &S,
+        offset: u64,
+        offset_size: u8,
+        length_size: u8,
     ) -> Result<Self, FormatError> {
         // FAHD signature(4) + version(1) + client_id(1) + element_size(1) +
         // max_nelmts_bits(1) + num_elements(length_size) + data_block_addr(offset_size) + checksum(4)
         let min_size = 4 + 1 + 1 + 1 + 1 + length_size as usize + offset_size as usize + 4;
-        ensure_len(file_data, offset, min_size)?;
+        let w = Window::read(file, offset, min_size)?;
+        w.ensure(0, min_size)?;
 
-        let d = &file_data[offset..];
+        let d: &[u8] = &w.bytes;
         if &d[0..4] != b"FAHD" {
             return Err(FormatError::ChunkedReadError(
                 "invalid Fixed Array header signature".into(),
@@ -130,7 +132,7 @@ impl FixedArrayHeader {
         pos += length_size as usize;
         let data_block_address = read_offset(d, pos, offset_size)?;
         pos += offset_size as usize;
-        verify_checksum(file_data, offset, offset + pos)?;
+        verify_checksum(&w, 0, pos)?;
 
         Ok(FixedArrayHeader {
             client_id,
@@ -156,15 +158,39 @@ pub fn read_fixed_array_chunks(
     chunk_dimensions: &[u32],
     element_size: u32,
     offset_size: u8,
+    length_size: u8,
+) -> Result<Vec<ChunkInfo>, FormatError> {
+    read_fixed_array_chunks_in(
+        &file_data,
+        header,
+        dataset_dims,
+        max_dims,
+        chunk_dimensions,
+        element_size,
+        offset_size,
+        length_size,
+    )
+}
+
+/// [`read_fixed_array_chunks`] over any [`Storage`]: one read of the data
+/// block's prefix, one of the whole data block (pages included).
+#[allow(clippy::too_many_arguments)]
+pub fn read_fixed_array_chunks_in<S: Storage + ?Sized>(
+    file: &S,
+    header: &FixedArrayHeader,
+    dataset_dims: &[u64],
+    max_dims: Option<&[u64]>,
+    chunk_dimensions: &[u32],
+    element_size: u32,
+    offset_size: u8,
     _length_size: u8,
 ) -> Result<Vec<ChunkInfo>, FormatError> {
-    let db_offset = header.data_block_address as usize;
+    let file_len = len_usize(file);
+    let db_offset = to_usize(header.data_block_address)?;
 
     // Parse data block header: FADB(4) + version(1) + client_id(1) + header_address(offset_size)
     let db_header_size = 4 + 1 + 1 + offset_size as usize;
-    ensure_len(file_data, db_offset, db_header_size)?;
+    let d = read_exact_at(file, db_offset as u64, db_header_size)?;
-
-    let d = &file_data[db_offset..];
     if &d[0..4] != b"FADB" {
         return Err(FormatError::ChunkedReadError(
             "invalid Fixed Array data block signature".into(),
@@ -174,11 +200,11 @@ pub fn read_fixed_array_chunks(
     // Elements start immediately after the data block prefix.
     let elements_start = db_offset + db_header_size;
 
-    let num_elements = header.num_elements as usize;
+    let num_elements = to_usize(header.num_elements)?;
     // A chunk index cannot describe more elements than the file has bytes (each
     // element occupies at least `offset_size` bytes). Reject a corrupt count
     // before it can drive a huge loop or overflow an offset computation.
-    if num_elements > file_data.len() {
+    if num_elements > file_len {
         return Err(FormatError::ChunkedReadError(
             "Fixed Array element count exceeds file size".into(),
         ));
@@ -208,11 +234,15 @@ pub fn read_fixed_array_chunks(
         chunk_dimensions.iter().map(|&d| d as u64).product::<u64>() * element_size as u64;
 
     let mut chunks = Vec::new();
-    let push_element =
+    // `rel` is relative to the data block, whose bytes are in `w`.
-        |i: usize, abs: usize, chunks: &mut Vec<ChunkInfo>| -> Result<(), FormatError> {
+    let push_element = |w: &Window<'_>,
+                        i: usize,
+                        rel: usize,
+                        chunks: &mut Vec<ChunkInfo>|
+     -> Result<(), FormatError> {
         if let Some((address, chunk_size, filter_mask)) = parse_fa_element(
-                file_data,
+            w,
-                abs,
+            rel,
             header.client_id,
             offset_size,
             header.element_size,
@@ -246,10 +276,16 @@ pub fn read_fixed_array_chunks(
 
     if !is_paged {
         // Non-paged: prefix, then `num_elements` elements packed directly,
-        // then a checksum over both.
+        // then a checksum over both. One window holds all of it (or ends at
-        verify_checksum(file_data, db_offset, elem_at(elements_start, num_elements)?)?;
+        // the end of the file), so its bounds checks are the whole-file ones.
+        let end = elem_at(elements_start, num_elements)?;
+        // The checksum's bounds check comes first: make it before reading.
+        #[cfg(feature = "checksum")]
+        Window::check_extent(file, db_offset as u64, end - db_offset, 4)?;
+        let w = Window::read(file, db_offset as u64, end.saturating_add(4) - db_offset)?;
+        verify_checksum(&w, 0, end - db_offset)?;
         for i in 0..num_elements {
-            push_element(i, elem_at(elements_start, i)?, &mut chunks)?;
+            push_element(&w, i, elem_at(elements_start, i)? - db_offset, &mut chunks)?;
         }
         return Ok(chunks);
     }
@@ -272,22 +308,40 @@ pub fn read_fixed_array_chunks(
         .and_then(|x| x.checked_add(4))
         .ok_or_else(stride_overflow)?;
 
-    if bitmap_start + bitmap_size > file_data.len() {
+    if bitmap_start + bitmap_size > file_len {
         return Err(FormatError::UnexpectedEof {
             expected: bitmap_start + bitmap_size,
-            available: file_data.len(),
+            available: file_len,
         });
     }
+    // The whole data block in one window when it is small: every page slot
+    // is at most `page_stride` bytes, so every position checked below lies
+    // inside it (or past the end of the file). A larger block is read as its
+    // prefix and bitmap, then each page in use on its own.
+    let block_len = (pages_start - db_offset).saturating_add(npages.saturating_mul(page_stride));
+    let whole = if block_len <= PAGED_BLOCK_ONE_READ_MAX {
+        Some(Window::read(file, db_offset as u64, block_len)?)
+    } else {
+        None
+    };
+    let head_w;
+    let head = match &whole {
+        Some(w) => w,
+        None => {
+            head_w = Window::read(file, db_offset as u64, pages_start - db_offset)?;
+            &head_w
+        }
+    };
     // The prefix and page bitmap are covered by their own checksum, and each
     // initialised page by one of its own.
-    verify_checksum(file_data, db_offset, bitmap_start + bitmap_size)?;
+    verify_checksum(head, 0, bitmap_start + bitmap_size - db_offset)?;
 
     for p in 0..npages {
         let page_first = p * page_nelmts; // < num_elements, cannot overflow
         let page_count = core::cmp::min(page_nelmts, num_elements - page_first);
 
         // Check the page-init bit (MSB-first within each byte).
-        let bit_byte = file_data[bitmap_start + p / 8];
+        let bit_byte = head.bytes[bitmap_start + p / 8 - db_offset];
         let bit_mask = 1u8 << (7 - (p % 8));
         if bit_byte & bit_mask == 0 {
             continue; // entire page unallocated
@@ -297,41 +351,49 @@ pub fn read_fixed_array_chunks(
             .checked_mul(page_stride)
             .and_then(|o| pages_start.checked_add(o))
             .ok_or_else(stride_overflow)?;
-        verify_checksum(file_data, page_off, elem_at(page_off, page_count)?)?;
+        let page_end = elem_at(page_off, page_count)?;
+        // `w` holds the page from `base` on (positions below are relative
+        // to it).
+        let page_w;
+        let (w, base) = match &whole {
+            Some(w) => (w, db_offset),
+            None => {
+                page_w =
+                    Window::read(file, page_off as u64, page_end.saturating_add(4) - page_off)?;
+                (&page_w, page_off)
+            }
+        };
+        verify_checksum(w, page_off - base, page_end - base)?;
         for e in 0..page_count {
-            push_element(page_first + e, elem_at(page_off, e)?, &mut chunks)?;
+            push_element(w, page_first + e, elem_at(page_off, e)? - base, &mut chunks)?;
         }
     }
 
     Ok(chunks)
 }
 
-/// Parse a single Fixed Array element at absolute file offset `abs`.
+/// Parse a single Fixed Array element at offset `abs` of the window `w`.
 ///
 /// Returns `Some((address, chunk_size, filter_mask))` for an allocated chunk, or
 /// `None` if the element is undefined (an unallocated chunk, address all-`0xFF`).
 fn parse_fa_element(
-    file_data: &[u8],
+    w: &Window<'_>,
     abs: usize,
     client_id: u8,
     offset_size: u8,
     element_size: u8,
     chunk_byte_size: u64,
-) -> Result<Option<(u64, u32, u32)>, FormatError> {
+) -> Result<Option<(u64, u64, u32)>, FormatError> {
     let os = offset_size as usize;
     if client_id == 0 {
         // Non-filtered: element is just the chunk address.
-        if abs + os > file_data.len() {
+        w.ensure(abs, os)?;
-            return Err(FormatError::UnexpectedEof {
+        let file_data: &[u8] = &w.bytes;
-                expected: abs + os,
-                available: file_data.len(),
-            });
-        }
         if is_undefined(file_data, abs, offset_size) {
             return Ok(None);
         }
         let address = read_offset(file_data, abs, offset_size)?;
-        Ok(Some((address, chunk_byte_size as u32, 0)))
+        Ok(Some((address, chunk_byte_size, 0)))
     } else {
         // Filtered: address(offset_size) + chunk_size(variable) + filter_mask(4)
         let es = element_size as usize;
@@ -341,17 +403,14 @@ fn parse_fa_element(
             ));
         }
         let chunk_size_bytes = es - os - 4;
-        if abs + es > file_data.len() {
+        w.ensure(abs, es)?;
-            return Err(FormatError::UnexpectedEof {
+        let file_data: &[u8] = &w.bytes;
-                expected: abs + es,
-                available: file_data.len(),
-            });
-        }
         if is_undefined(file_data, abs, offset_size) {
             return Ok(None);
         }
         let address = read_offset(file_data, abs, offset_size)?;
-        let chunk_size = read_variable_length(&file_data[abs + os..], chunk_size_bytes)?;
+        let chunk_size =
+            read_variable_length(&file_data[abs + os..abs + es - 4], chunk_size_bytes)?;
         let fm_off = abs + os + chunk_size_bytes;
         let filter_mask = u32::from_le_bytes([
             file_data[fm_off],
@@ -359,7 +418,7 @@ fn parse_fa_element(
             file_data[fm_off + 2],
             file_data[fm_off + 3],
         ]);
-        Ok(Some((address, chunk_size as u32, filter_mask)))
+        Ok(Some((address, chunk_size, filter_mask)))
     }
 }
 
@@ -629,7 +688,7 @@ mod tests {
             assert_eq!(c.address, base_addr + i as u64 * chunk_byte_size as u64);
             assert_eq!(c.offsets, vec![i as u64 * 20]);
             assert_eq!(c.filter_mask, 0);
-            assert_eq!(c.chunk_size, chunk_byte_size as u32);
+            assert_eq!(c.chunk_size, chunk_byte_size as u64);
         }
     }
 
@@ -813,4 +872,126 @@ mod tests {
             .collect();
         assert_eq!(got, expect);
     }
+
+    /// A fixed array (header at 0x100, data block at 0x200) of `n` chunks,
+    /// filtered or not, paged when `n` exceeds `1 << page_bits`; every
+    /// page initialised except page 1.
+    fn build_fixed_array(n: usize, filtered: bool, page_bits: u8) -> Vec<u8> {
+        let os = 8usize;
+        let es = if filtered { os + 4 + 4 } else { os };
+        let (fahd, db) = (0x100usize, 0x200usize);
+        let mut f = vec![0u8; 0x2000];
+        f[fahd..fahd + 4].copy_from_slice(b"FAHD");
+        f[fahd + 5] = u8::from(filtered);
+        f[fahd + 6] = es as u8;
+        f[fahd + 7] = page_bits;
+        f[fahd + 8..fahd + 16].copy_from_slice(&(n as u64).to_le_bytes());
+        f[fahd + 16..fahd + 24].copy_from_slice(&(db as u64).to_le_bytes());
+        stamp_checksum(&mut f, fahd, fahd + 24);
+        f[db..db + 4].copy_from_slice(b"FADB");
+        f[db + 5] = u8::from(filtered);
+        f[db + 6..db + 14].copy_from_slice(&(fahd as u64).to_le_bytes());
+        let elems = db + 6 + os;
+        let write = |f: &mut Vec<u8>, at: usize, i: usize| {
+            let addr = if i == 2 {
+                u64::MAX
+            } else {
+                0x1000 + i as u64 * 0x100
+            };
+            f[at..at + os].copy_from_slice(&addr.to_le_bytes());
+            if filtered {
+                f[at + os..at + os + 4].copy_from_slice(&(100 + i as u32).to_le_bytes());
+                f[at + os + 4..at + os + 8].copy_from_slice(&(i as u32 & 1).to_le_bytes());
+            }
+        };
+        let page = 1usize << page_bits;
+        if n <= page {
+            for i in 0..n {
+                write(&mut f, elems + i * es, i);
+            }
+            stamp_checksum(&mut f, db, elems + n * es);
+        } else {
+            let npages = n.div_ceil(page);
+            let bitmap = npages.div_ceil(8);
+            for p in 0..npages {
+                if p != 1 {
+                    f[elems + p / 8] |= 0x80 >> (p % 8);
+                }
+            }
+            stamp_checksum(&mut f, db, elems + bitmap);
+            let pages_start = elems + bitmap + 4;
+            for p in (0..npages).filter(|&p| p != 1) {
+                let at = pages_start + p * (page * es + 4);
+                let count = page.min(n - p * page);
+                for e in 0..count {
+                    write(&mut f, at + e * es, p * page + e);
+                }
+                stamp_checksum(&mut f, at, at + count * es);
+            }
+        }
+        f
+    }
+
+    /// Non-paged and paged, filtered and unfiltered arrays, cut at every
+    /// length through the data block and with a damaged byte, read
+    /// identically through a `read_at`-only storage.
+    #[test]
+    fn storage_reads_match_slice_reads() {
+        use crate::storage::CountingStorage;
+        for (n, filtered, bits) in [(3, false, 10), (3, true, 10), (11, false, 2), (11, true, 2)] {
+            let full = build_fixed_array(n, filtered, bits);
+            let es = if filtered { 16 } else { 8 };
+            let dims = [n as u64 * 20];
+            let h = FixedArrayHeader::parse(&full, 0x100, 8, 8).unwrap();
+            let chunks = read_fixed_array_chunks(&full, &h, &dims, None, &[20], 8, 8, 8).unwrap();
+            // Chunk 2 is unallocated, and so is page 1 of a paged array.
+            let expect = if n > 4 { n - 1 - 4 } else { n - 1 };
+            assert_eq!(chunks.len(), expect);
+            let mut files = Vec::new();
+            for cut in (0x100..0x200 + 40 + n * (es + 4) + 16).step_by(3) {
+                files.push(full[..cut].to_vec());
+            }
+            for at in [0x104, 0x210, 0x21a, 0x230] {
+                let mut damaged = full.clone();
+                damaged[at] ^= 1;
+                files.push(damaged);
+            }
+            files.push(full);
+            for f in files {
+                let storage = CountingStorage::new(f.clone());
+                let want = FixedArrayHeader::parse(&f, 0x100, 8, 8);
+                let got = FixedArrayHeader::parse_in(&storage, 0x100, 8, 8);
+                assert_eq!(format!("{got:?}"), format!("{want:?}"));
+                let Ok(h) = want else { continue };
+                let want = read_fixed_array_chunks(&f, &h, &dims, None, &[20], 8, 8, 8);
+                let got = read_fixed_array_chunks_in(&storage, &h, &dims, None, &[20], 8, 8, 8);
+                assert_eq!(format!("{got:?}"), format!("{want:?}"), "{} bytes", f.len());
+            }
+        }
+    }
+
+    /// A header whose element count stretches its data block (one checksum
+    /// over the whole block) far past the end of a 16 MiB file: the
+    /// checksum's bounds check fails before the block is read, with the
+    /// slice read's error.
+    #[cfg(feature = "checksum")]
+    #[test]
+    fn oversized_block_fails_before_reading() {
+        use crate::storage::CountingStorage;
+        let mut f = build_fixed_array(3, false, 10);
+        f.resize(16 << 20, 0);
+        let mut h = FixedArrayHeader::parse(&f, 0x100, 8, 8).unwrap();
+        h.max_nelmts_bits = 30;
+        h.num_elements = 4 << 20;
+        let dims = [h.num_elements * 20];
+        let want = read_fixed_array_chunks(&f, &h, &dims, None, &[20], 8, 8, 8);
+        assert!(
+            matches!(want, Err(FormatError::UnexpectedEof { .. })),
+            "{want:?}"
+        );
+        let storage = CountingStorage::new(f);
+        let got = read_fixed_array_chunks_in(&storage, &h, &dims, None, &[20], 8, 8, 8);
+        assert_eq!(format!("{got:?}"), format!("{want:?}"));
+        assert!(storage.bytes_read() < 64, "{} bytes", storage.bytes_read());
+    }
 }

crates/clawhdf5-format/src/fractal_heap.rs

@@ -6,9 +6,11 @@ use alloc::{format, vec::Vec};
 #[cfg(feature = "checksum")]
 use byteorder::{ByteOrder, LittleEndian};
 
-use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records};
+use crate::addr::to_usize;
+use crate::btree_v2::{BTreeV2Header, find_btree_v2_records_in};
 use crate::error::FormatError;
 use crate::filter_pipeline::FilterPipeline;
+use crate::storage::{Storage, Window, len_usize, read_exact_at, read_upto};
 
 /// Parsed fractal heap header (signature "FRHP").
 #[derive(Debug, Clone)]
@@ -130,6 +132,9 @@ fn heap_id_type(first: u8) -> Result<u8, FormatError> {
 const BTREE_HUGE_INDIRECT: u8 = 1;
 const BTREE_HUGE_INDIRECT_FILTERED: u8 = 2;
 
+/// Most child entries [`FractalHeapHeader::hint_managed_blocks`] walks.
+const MAX_HINTED_BLOCKS: usize = 4096;
+
 impl FractalHeapHeader {
     /// Parse a fractal heap header at the given offset.
     pub fn parse(
@@ -138,6 +143,36 @@ impl FractalHeapHeader {
         offset_size: u8,
         length_size: u8,
     ) -> Result<FractalHeapHeader, FormatError> {
+        Self::parse_in(file_data, offset as u64, offset_size, length_size)
+    }
+
+    /// [`Self::parse`] over any [`Storage`]: one read of the header (two
+    /// when it holds an I/O filter pipeline).
+    pub fn parse_in<S: Storage + ?Sized>(
+        file: &S,
+        offset: u64,
+        offset_size: u8,
+        length_size: u8,
+    ) -> Result<FractalHeapHeader, FormatError> {
+        // Every field up to the checksum, without and with the filter
+        // information; the window holds all of it (or ends at the end of
+        // the file), so its bounds checks are the whole-file ones.
+        let (os, ls) = (usize::from(offset_size), usize::from(length_size));
+        let unfiltered_len = 26 + 12 * ls + 3 * os;
+        let mut w = Window::read(file, offset, unfiltered_len)?;
+        if w.bytes.len() == unfiltered_len {
+            let filter_len = usize::from(u16::from_le_bytes([w.bytes[7], w.bytes[8]]));
+            if filter_len > 0 {
+                w = Window::read(file, offset, unfiltered_len + ls + 4 + filter_len)?;
+            }
+        }
+        let ensure_len = |_: &[u8], pos: usize, needed: usize| w.ensure(pos, needed);
+        let read_offset = |_: &[u8], pos: usize, size: u8| {
+            w.ensure(pos, usize::from(size))?;
+            read_offset(&w.bytes, pos, size)
+        };
+        let file_data: &[u8] = &w.bytes;
+        let offset = 0usize;
         ensure_len(file_data, offset, 5)?;
         if &file_data[offset..offset + 4] != b"FRHP" {
             return Err(FormatError::InvalidFractalHeapSignature);
@@ -148,9 +183,6 @@ impl FractalHeapHeader {
             return Err(FormatError::InvalidFractalHeapVersion(version));
         }
 
-        let os = offset_size as usize;
-        let ls = length_size as usize;
-
         let mut pos = offset + 5;
         ensure_len(file_data, pos, 2)?;
         let heap_id_length = u16::from_le_bytes([file_data[pos], file_data[pos + 1]]);
@@ -354,6 +386,17 @@ impl FractalHeapHeader {
         id_bytes: &[u8],
         offset_size: u8,
     ) -> Result<Vec<u8>, FormatError> {
+        self.read_managed_object_in(file_data, id_bytes, offset_size)
+    }
+
+    /// [`Self::read_managed_object`] over any [`Storage`].
+    pub fn read_managed_object_in<S: Storage + ?Sized>(
+        &self,
+        file_data: &S,
+        id_bytes: &[u8],
+        offset_size: u8,
+    ) -> Result<Vec<u8>, FormatError> {
+        crate::lookup_stats::heap_object_read();
         let Some(&first) = id_bytes.first() else {
             return Err(FormatError::UnexpectedEof {
                 expected: 1,
@@ -383,7 +426,11 @@ impl FractalHeapHeader {
     }
 
     /// Read a huge object (heap ID type 1).
-    fn read_huge_object(&self, file_data: &[u8], id: &[u8]) -> Result<Vec<u8>, FormatError> {
+    fn read_huge_object<S: Storage + ?Sized>(
+        &self,
+        file: &S,
+        id: &[u8],
+    ) -> Result<Vec<u8>, FormatError> {
         let os = usize::from(self.offset_size);
         let ls = usize::from(self.length_size);
         // (address, stored length, filter mask, decoded length); the last two
@@ -414,18 +461,17 @@ impl FractalHeapHeader {
             let key_len = (usize::from(self.heap_id_length).saturating_sub(1)).min(8);
             ensure_len(id, 1, key_len)?;
             let key = le_uint(&id[1..1 + key_len]);
-            self.find_huge_record(file_data, key)?
+            self.find_huge_record(file, key)?
         };
 
         let start = usize::try_from(addr).map_err(|_| heap_error("huge object address"))?;
         let len = usize::try_from(stored_len).map_err(|_| heap_error("huge object length"))?;
-        ensure_len(file_data, start, len)?;
+        let stored = read_exact_at(file, start as u64, len)?;
-        let stored = &file_data[start..start + len];
         match &self.filter_pipeline {
-            None => Ok(stored.to_vec()),
+            None => Ok(stored.into_owned()),
             Some(pipeline) => {
                 let mem = usize::try_from(mem_len).map_err(|_| heap_error("huge object size"))?;
-                let out = crate::filters::decompress_chunk_masked(stored, pipeline, mem, 1, mask)?;
+                let out = crate::filters::decompress_chunk_masked(&stored, pipeline, mem, 1, mask)?;
                 if out.len() != mem {
                     return Err(heap_error("filtered huge object decoded to the wrong size"));
                 }
@@ -436,9 +482,9 @@ impl FractalHeapHeader {
 
     /// Look up huge object `key` in the huge-object v2 B-tree, returning
     /// (address, stored length, filter mask, decoded length).
-    fn find_huge_record(
+    fn find_huge_record<S: Storage + ?Sized>(
         &self,
-        file_data: &[u8],
+        file: &S,
         key: u64,
     ) -> Result<(u64, u64, u32, u64), FormatError> {
         if is_undefined(self.huge_btree_address, self.offset_size) {
@@ -446,9 +492,9 @@ impl FractalHeapHeader {
                 "huge object ID but the heap has no huge-object index",
             ));
         }
-        let hdr = BTreeV2Header::parse(
+        let hdr = BTreeV2Header::parse_in(
-            file_data,
+            file,
-            self.huge_btree_address as usize,
+            self.huge_btree_address,
             self.offset_size,
             self.length_size,
         )?;
@@ -463,8 +509,12 @@ impl FractalHeapHeader {
         if hdr.tree_type != expected_type || usize::from(hdr.record_size) < rec_len {
             return Err(heap_error("unexpected huge-object B-tree record type"));
         }
-        let records =
+        // Records are ordered by ID (the last field): descend to the ones
-            collect_btree_v2_records(file_data, &hdr, self.offset_size, self.length_size)?;
+        // equal to `key` instead of reading the whole index.
+        let id_at = rec_len - ls;
+        let records = find_btree_v2_records_in(file, &hdr, self.offset_size, &mut |r| {
+            le_uint(&r[id_at..id_at + ls]).cmp(&key)
+        })?;
         for rec in &records {
             let d = &rec.data;
             if d.len() < rec_len {
@@ -513,9 +563,9 @@ impl FractalHeapHeader {
     }
 
     /// Read a managed object (heap ID type 0).
-    fn read_heap_managed(
+    fn read_heap_managed<S: Storage + ?Sized>(
         &self,
-        file_data: &[u8],
+        file_data: &S,
         id_bytes: &[u8],
         offset_size: u8,
     ) -> Result<Vec<u8>, FormatError> {
@@ -533,24 +583,24 @@ impl FractalHeapHeader {
             self.read_from_direct_block(
                 file_data,
                 DirectBlock {
-                    addr: self.root_block_address as usize,
+                    addr: to_usize(self.root_block_address)?,
                     size: self.starting_block_size,
                     heap_offset: 0,
                     filtered_size: self.root_direct_block_filtered_size,
                     filter_mask: self.root_direct_block_filter_mask,
                 },
                 heap_offset,
-                obj_len as usize,
+                to_usize(obj_len)?,
             )
         } else {
             // Root is an indirect block — limit recursion to 64 levels
             self.read_from_indirect_block(
                 file_data,
-                self.root_block_address as usize,
+                to_usize(self.root_block_address)?,
                 self.current_rows_in_root_indirect_block,
                 0, // block offset
                 heap_offset,
-                obj_len as usize,
+                to_usize(obj_len)?,
                 offset_size,
                 64, // max recursion depth
             )
@@ -563,27 +613,27 @@ impl FractalHeapHeader {
     /// header), so we just add it to the block address minus the block's heap
     /// offset. A filtered heap stores each direct block (header included)
     /// through its filter pipeline, so the block is decoded first.
-    fn read_from_direct_block(
+    fn read_from_direct_block<S: Storage + ?Sized>(
         &self,
-        file_data: &[u8],
+        file: &S,
         block: DirectBlock,
         target_offset: u64,
         length: usize,
     ) -> Result<Vec<u8>, FormatError> {
         if target_offset < block.heap_offset {
             return Err(FormatError::UnexpectedEof {
-                expected: block.heap_offset as usize,
+                expected: to_usize(block.heap_offset)?,
-                available: target_offset as usize,
+                available: to_usize(target_offset)?,
             });
         }
-        let local_offset = (target_offset - block.heap_offset) as usize;
+        let local_offset = to_usize(target_offset - block.heap_offset)?;
         if let Some(pipeline) = &self.filter_pipeline {
             let stored_len = usize::try_from(block.filtered_size)
                 .map_err(|_| heap_error("direct block size"))?;
             let size = usize::try_from(block.size).map_err(|_| heap_error("direct block size"))?;
-            ensure_len(file_data, block.addr, stored_len)?;
+            let stored = read_exact_at(file, block.addr as u64, stored_len)?;
             let decoded = crate::filters::decompress_chunk_masked(
-                &file_data[block.addr..block.addr + stored_len],
+                &stored,
                 pipeline,
                 size,
                 1,
@@ -597,17 +647,16 @@ impl FractalHeapHeader {
             .checked_add(local_offset)
             .ok_or(FormatError::UnexpectedEof {
                 expected: usize::MAX,
-                available: file_data.len(),
+                available: len_usize(file),
             })?;
-        ensure_len(file_data, pos, length)?;
+        Ok(read_exact_at(file, pos as u64, length)?.into_owned())
-        Ok(file_data[pos..pos + length].to_vec())
     }
 
     /// Read an object by traversing an indirect block to find the right direct block.
     #[allow(clippy::too_many_arguments)]
-    fn read_from_indirect_block(
+    fn read_from_indirect_block<S: Storage + ?Sized>(
         &self,
-        file_data: &[u8],
+        file: &S,
         iblock_addr: usize,
         nrows: u16,
         iblock_heap_offset: u64,
@@ -621,29 +670,275 @@ impl FractalHeapHeader {
                 "fractal heap: maximum recursion depth exceeded".into(),
             ));
         }
-        // Parse indirect block header
+        let nrows_usize = nrows as usize;
-        ensure_len(file_data, iblock_addr, 4)?;
+
-        if &file_data[iblock_addr..iblock_addr + 4] != b"FHIB" {
+        // The block up to its last child entry. The walk below reads
-            return Err(FormatError::InvalidFractalHeapSignature);
+        // entries in order and stops at the one covering the target, which
+        // the geometry alone locates, so the first window ends there: a
+        // header claiming a huge table costs a read of the entries in front
+        // of the target, not of the rest of the file. Only when that entry
+        // is unallocated (or none covers the target) does the walk go on,
+        // over the whole block. Either window holds what it was asked for or
+        // ends at the end of the file, so its bounds checks are the
+        // whole-file ones.
+        let layout = self.indirect_layout(nrows_usize, offset_size);
+        let block_len = layout.len_upto(layout.all_entries);
+        let target_entry = self.indirect_entry_for(nrows_usize, iblock_heap_offset, target_offset);
+        let first_len = target_entry.map_or(block_len, |i| {
+            layout.len_upto(i.saturating_add(1)).min(block_len)
+        });
+        let mut next = self.walk_indirect_block(
+            &Window::read(file, iblock_addr as u64, first_len)?,
+            nrows_usize,
+            iblock_heap_offset,
+            target_offset,
+            offset_size,
+            target_entry.map_or(usize::MAX, |i| i.saturating_add(1)),
+        )?;
+        if next.is_none() && first_len < block_len {
+            next = self.walk_indirect_block(
+                &Window::read(file, iblock_addr as u64, block_len)?,
+                nrows_usize,
+                iblock_heap_offset,
+                target_offset,
+                offset_size,
+                usize::MAX,
+            )?;
+        }
+        match next {
+            Some(IndirectChild::Direct(block)) => {
+                self.read_from_direct_block(file, block, target_offset, length)
+            }
+            Some(IndirectChild::Indirect {
+                addr,
+                nrows,
+                heap_offset,
+            }) => self.read_from_indirect_block(
+                file,
+                addr,
+                nrows,
+                heap_offset,
+                target_offset,
+                length,
+                offset_size,
+                depth_remaining - 1,
+            ),
+            None => Err(FormatError::UnexpectedEof {
+                expected: to_usize(target_offset)?.saturating_add(length),
+                available: len_usize(file),
+            }),
+        }
     }
 
+    /// Where the child entries of an indirect block of `nrows` rows are.
+    fn indirect_layout(&self, nrows: usize, offset_size: u8) -> IndirectLayout {
         let block_offset_bytes = (self.max_heap_size as usize).div_ceil(8);
-        let iblock_header = 5 + offset_size as usize + block_offset_bytes;
-        let mut pos = iblock_addr + iblock_header;
-
-        let tw = self.table_width as u64;
-        let nrows_usize = nrows as usize;
-        let mut current_heap_offset = iblock_heap_offset;
-
         // Rows below max_direct_rows hold direct blocks; rows at/above hold
         // child indirect blocks. (NOT the FRHP "starting rows" field.)
         let start_indirect = self.max_direct_rows();
-        let max_direct_rows = nrows_usize.min(start_indirect);
+        let direct_entries = nrows
+            .min(start_indirect)
+            .saturating_mul(usize::from(self.table_width));
+        IndirectLayout {
+            header: 5 + usize::from(offset_size) + block_offset_bytes,
+            direct_entry: usize::from(offset_size)
+                + if self.filter_pipeline.is_some() {
+                    usize::from(self.length_size) + 4
+                } else {
+                    0
+                },
+            direct_entries,
+            indirect_entry: usize::from(offset_size),
+            all_entries: direct_entries.saturating_add(
+                nrows
+                    .saturating_sub(start_indirect)
+                    .saturating_mul(usize::from(self.table_width)),
+            ),
+        }
+    }
+
+    /// Hint the heap's root block (see [`Storage::hint`]): every managed
+    /// object is read through it, and a storage that fetches between
+    /// attempts can fetch it along with whatever else the attempt missed
+    /// (the name index read before any object, say).
+    pub fn hint_root_block<S: Storage + ?Sized>(&self, file: &S) {
+        if is_undefined(self.root_block_address, self.offset_size) {
+            return;
+        }
+        let len = if self.current_rows_in_root_indirect_block == 0 {
+            if self.filter_pipeline.is_some() {
+                self.root_direct_block_filtered_size
+            } else {
+                self.starting_block_size
+            }
+        } else {
+            let layout = self.indirect_layout(
+                usize::from(self.current_rows_in_root_indirect_block),
+                self.offset_size,
+            );
+            layout.len_upto(layout.all_entries) as u64
+        };
+        file.hint(
+            self.root_block_address,
+            usize::try_from(len).unwrap_or(usize::MAX),
+        );
+    }
+
+    /// Hint every managed direct block of the heap (see
+    /// [`Storage::hint`]), for a caller about to read all of its objects (a
+    /// dense group's listing). The indirect blocks leading to them are read
+    /// here, as every object read goes through them, a few levels deep and
+    /// up to [`MAX_HINTED_BLOCKS`] entries; direct blocks are only hinted.
+    /// Nothing is returned and no error: a storage that has the file in
+    /// memory skips it, and the objects are read (and checked) as before.
+    pub fn hint_managed_blocks<S: Storage + ?Sized>(&self, file: &S) {
+        if file.as_contiguous().is_some()
+            || is_undefined(self.root_block_address, self.offset_size)
+            || self.current_rows_in_root_indirect_block == 0
+        {
+            // A direct root block is what `hint_root_block` hints.
+            return;
+        }
+        let mut budget = MAX_HINTED_BLOCKS;
+        self.hint_indirect_block(
+            file,
+            self.root_block_address,
+            usize::from(self.current_rows_in_root_indirect_block),
+            0,
+            &mut budget,
+        );
+    }
+
+    fn hint_indirect_block<S: Storage + ?Sized>(
+        &self,
+        file: &S,
+        addr: u64,
+        nrows: usize,
+        depth: usize,
+        budget: &mut usize,
+    ) {
+        let os = self.offset_size;
+        let layout = self.indirect_layout(nrows, os);
+        let len = layout.len_upto(layout.all_entries);
+        if depth > 4 || len > 1 << 20 {
+            return;
+        }
+        let Ok(bytes) = read_upto(file, addr, len) else {
+            return;
+        };
+        if bytes.len() < len || bytes.get(..4) != Some(b"FHIB".as_slice()) {
+            return;
+        }
+        let start_indirect = self.max_direct_rows();
+        let mut pos = layout.header;
+        for row in 0..nrows {
+            for _ in 0..self.table_width {
+                let Some(left) = budget.checked_sub(1) else {
+                    return;
+                };
+                *budget = left;
+                let Ok(child) = read_offset(&bytes, pos, os) else {
+                    return;
+                };
+                if row < start_indirect {
+                    let size = if self.filter_pipeline.is_some() {
+                        match read_offset(&bytes, pos + usize::from(os), self.length_size) {
+                            Ok(n) => n,
+                            Err(_) => return,
+                        }
+                    } else {
+                        self.block_size_for_row(row)
+                    };
+                    pos += layout.direct_entry;
+                    if !is_undefined(child, os) {
+                        file.hint(child, usize::try_from(size).unwrap_or(usize::MAX));
+                    }
+                } else {
+                    pos += layout.indirect_entry;
+                    if !is_undefined(child, os) {
+                        let rows = self.rows_for_size(self.block_size_for_row(row));
+                        self.hint_indirect_block(file, child, usize::from(rows), depth + 1, budget);
+                    }
+                }
+            }
+        }
+    }
+
+    /// Which child entry of an indirect block (numbered in walk order:
+    /// direct rows, then indirect rows) covers `target_offset`, from the
+    /// doubling-table geometry alone — the entry
+    /// [`Self::walk_indirect_block`] stops at if it is allocated. `None`
+    /// when no entry does.
+    fn indirect_entry_for(&self, nrows: usize, heap_offset: u64, target: u64) -> Option<usize> {
+        // The walk adds block sizes with saturation; in u128 the same test
+        // is `cur <= target < cur + size` without it (a target of u64::MAX
+        // is never inside a saturated range).
+        if target == u64::MAX {
+            return None;
+        }
+        let (tw, target) = (u128::from(self.table_width), u128::from(target));
+        let mut cur = u128::from(heap_offset);
+        let mut before = 0usize;
+        for row in 0..nrows {
+            if target < cur {
+                return None;
+            }
+            // Direct and indirect rows alike span this row's block size per
+            // entry.
+            let size = u128::from(self.block_size_for_row(row));
+            let span = size * tw;
+            if size > 0 && target < cur + span {
+                let col = usize::try_from((target - cur) / size).ok()?;
+                return before.checked_add(col);
+            }
+            cur += span;
+            before = before.saturating_add(self.table_width as usize);
+        }
+        None
+    }
+
+    /// Walk an indirect block's child entries in order, in the window `w`
+    /// (the block from its signature on), and return the allocated child
+    /// covering `target_offset`, or `None` when no entry among the first
+    /// `limit` does.
+    fn walk_indirect_block(
+        &self,
+        w: &Window<'_>,
+        nrows: usize,
+        iblock_heap_offset: u64,
+        target_offset: u64,
+        offset_size: u8,
+        limit: usize,
+    ) -> Result<Option<IndirectChild>, FormatError> {
+        let ensure_len = |_: &[u8], pos: usize, needed: usize| w.ensure(pos, needed);
+        let read_offset = |_: &[u8], pos: usize, size: u8| {
+            w.ensure(pos, usize::from(size))?;
+            read_offset(&w.bytes, pos, size)
+        };
+        let file_data: &[u8] = &w.bytes;
+        let block_offset_bytes = (self.max_heap_size as usize).div_ceil(8);
+        let iblock_header = 5 + offset_size as usize + block_offset_bytes;
+        let tw = self.table_width as u64;
+        let mut current_heap_offset = iblock_heap_offset;
+        let start_indirect = self.max_direct_rows();
+        let max_direct_rows = nrows.min(start_indirect);
+        let mut walked = 0usize;
+
+        // Parse indirect block header
+        ensure_len(file_data, 0, 4)?;
+        if &file_data[..4] != b"FHIB" {
+            return Err(FormatError::InvalidFractalHeapSignature);
+        }
+        let mut pos = iblock_header;
 
         for row in 0..max_direct_rows {
             let block_size = self.block_size_for_row(row);
 
             for _col in 0..tw {
+                if walked == limit {
+                    return Ok(None);
+                }
+                walked += 1;
                 let child_addr = read_offset(file_data, pos, offset_size)?;
                 pos += offset_size as usize;
 
@@ -670,18 +965,13 @@ impl FractalHeapHeader {
                     && target_offset >= current_heap_offset
                     && target_offset < block_end
                 {
-                    return self.read_from_direct_block(
+                    return Ok(Some(IndirectChild::Direct(DirectBlock {
-                        file_data,
+                        addr: to_usize(child_addr)?,
-                        DirectBlock {
-                            addr: child_addr as usize,
                         size: block_size,
                         heap_offset: current_heap_offset,
                         filtered_size,
                         filter_mask,
-                        },
+                    })));
-                        target_offset,
-                        length,
-                    );
                 }
                 current_heap_offset = block_end;
             }
@@ -690,11 +980,15 @@ impl FractalHeapHeader {
         // Rows at and above `start_indirect` hold child indirect blocks. A
         // child in row r spans exactly that row's block size of heap space,
         // so it has as many rows as a table of that total size needs.
-        for row in start_indirect..nrows_usize {
+        for row in start_indirect..nrows {
             let child_space = self.block_size_for_row(row);
             let child_nrows = self.rows_for_size(child_space);
 
             for _col in 0..tw {
+                if walked == limit {
+                    return Ok(None);
+                }
+                walked += 1;
                 let child_addr = read_offset(file_data, pos, offset_size)?;
                 pos += offset_size as usize;
 
@@ -703,25 +997,16 @@ impl FractalHeapHeader {
                     && target_offset >= current_heap_offset
                     && target_offset < block_end
                 {
-                    return self.read_from_indirect_block(
+                    return Ok(Some(IndirectChild::Indirect {
-                        file_data,
+                        addr: to_usize(child_addr)?,
-                        child_addr as usize,
+                        nrows: child_nrows,
-                        child_nrows,
+                        heap_offset: current_heap_offset,
-                        current_heap_offset,
+                    }));
-                        target_offset,
-                        length,
-                        offset_size,
-                        depth_remaining - 1,
-                    );
                 }
                 current_heap_offset = block_end;
             }
         }
-
+        Ok(None)
-        Err(FormatError::UnexpectedEof {
-            expected: target_offset as usize + length,
-            available: file_data.len(),
-        })
     }
 
     /// Number of rows in the doubling table whose block size is at most the
@@ -763,8 +1048,43 @@ impl FractalHeapHeader {
     }
 }
 
+/// Where an indirect block's child entries are: after its header, the
+/// direct blocks' entries (address, and for a filtered heap the stored
+/// size and filter mask), then the child indirect blocks' (address).
+struct IndirectLayout {
+    header: usize,
+    direct_entry: usize,
+    direct_entries: usize,
+    indirect_entry: usize,
+    all_entries: usize,
+}
+
+impl IndirectLayout {
+    /// Bytes from the block's start to the end of its first `n` entries.
+    fn len_upto(&self, n: usize) -> usize {
+        let direct = n.min(self.direct_entries);
+        self.header
+            .saturating_add(direct.saturating_mul(self.direct_entry))
+            .saturating_add(
+                n.min(self.all_entries)
+                    .saturating_sub(direct)
+                    .saturating_mul(self.indirect_entry),
+            )
+    }
+}
+
 /// A managed direct block's location, extent and (for a filtered heap) its
 /// stored size and filter mask.
+/// The child of an indirect block that covers a heap offset.
+enum IndirectChild {
+    Direct(DirectBlock),
+    Indirect {
+        addr: usize,
+        nrows: u16,
+        heap_offset: u64,
+    },
+}
+
 struct DirectBlock {
     addr: usize,
     size: u64,
@@ -1024,4 +1344,152 @@ mod tests {
         let id = [0x40u8, 0, 0, 0, 0, 0, 0];
         assert!(hdr.read_managed_object(&file_data, &id, 8).is_err());
     }
+
+    /// Headers, and managed (in a direct root and through an indirect
+    /// root), huge and tiny objects read identically through a
+    /// `read_at`-only storage, for every truncation of the file.
+    #[test]
+    fn storage_reads_match_slice_reads() {
+        use crate::storage::CountingStorage;
+        let (mut file, header_end) = build_simple_heap(8, 8);
+        // An indirect root block at 600: row 0 holds the direct block at
+        // 256, then three undefined blocks.
+        file[600..604].copy_from_slice(b"FHIB");
+        let mut at = 600 + 5 + 8 + 2;
+        for addr in [256u64, u64::MAX, u64::MAX, u64::MAX] {
+            file[at..at + 8].copy_from_slice(&addr.to_le_bytes());
+            at += 8;
+        }
+        file[900..905].copy_from_slice(b"huge!");
+        let managed_id = |offset: u64, len: u64| {
+            let payload = offset | (len << 16);
+            let mut id = vec![0u8];
+            id.extend_from_slice(&payload.to_le_bytes()[..6]);
+            id
+        };
+        let mut huge = vec![0x10u8];
+        huge.extend_from_slice(&900u64.to_le_bytes());
+        huge.extend_from_slice(&5u64.to_le_bytes());
+        let ids = [
+            managed_id(15, 13),
+            managed_id(15, 200),
+            managed_id(130, 4),
+            huge,
+            vec![0x22, b'a', b'b', b'c', 0, 0, 0],
+        ];
+        let mut cuts: Vec<usize> = (0..=header_end + 1).collect();
+        cuts.extend([256, 260, 271, 280, 600, 610, 620, 640, 900, 903, file.len()]);
+        for cut in cuts {
+            let f = &file[..cut];
+            let storage = CountingStorage::new(f.to_vec());
+            let want = FractalHeapHeader::parse(f, 0, 8, 8);
+            let got = FractalHeapHeader::parse_in(&storage, 0, 8, 8);
+            assert_eq!(format!("{got:?}"), format!("{want:?}"), "cut {cut}");
+            let Ok(direct) = want else { continue };
+            let mut indirect = direct.clone();
+            indirect.root_block_address = 600;
+            indirect.current_rows_in_root_indirect_block = 1;
+            let mut huge_ids = direct.clone();
+            huge_ids.heap_id_length = 17;
+            for hdr in [&direct, &indirect, &huge_ids] {
+                for id in &ids {
+                    assert_eq!(
+                        hdr.read_managed_object_in(&storage, id, 8),
+                        hdr.read_managed_object(f, id, 8),
+                        "cut {cut}"
+                    );
+                }
+            }
+        }
+    }
+
+    /// A header claiming a huge doubling table (width 0xFFFF, 0xFFFF rows in
+    /// the root indirect block) in a 16 MiB file: reading an object from the
+    /// table's first block reads the entries up to it, not the rest of the
+    /// file, and gives what the slice read gives. When the covering entry is
+    /// unallocated the walk goes on over the whole block, still identically.
+    #[test]
+    fn huge_table_claims_read_only_what_the_walk_needs() {
+        use crate::storage::CountingStorage;
+        let (mut file, _) = build_simple_heap(8, 8);
+        file.resize(16 << 20, 0);
+        file[600..604].copy_from_slice(b"FHIB");
+        let first_entry = 600 + 5 + 8 + 2;
+        file[first_entry..first_entry + 8].copy_from_slice(&256u64.to_le_bytes());
+        let mut hdr = FractalHeapHeader::parse(&file, 0, 8, 8).unwrap();
+        hdr.table_width = 0xFFFF;
+        hdr.root_block_address = 600;
+        hdr.current_rows_in_root_indirect_block = 0xFFFF;
+        let managed_id = |offset: u64, len: u64| {
+            let payload = offset | (len << 16);
+            let mut id = vec![0u8];
+            id.extend_from_slice(&payload.to_le_bytes()[..6]);
+            id
+        };
+        let storage = CountingStorage::new(file.clone());
+        let id = managed_id(15, 13);
+        let want = hdr.read_managed_object(&file, &id, 8);
+        assert!(want.is_ok(), "{want:?}");
+        storage.reset();
+        assert_eq!(hdr.read_managed_object_in(&storage, &id, 8), want);
+        assert!(
+            storage.bytes_read() < 1024,
+            "{} bytes in {} reads",
+            storage.bytes_read(),
+            storage.reads()
+        );
+        // The second entry (heap offsets 128..256) is unallocated (zero is
+        // not the undefined address, so make it all ones).
+        file[first_entry + 8..first_entry + 16].fill(0xFF);
+        let storage = CountingStorage::new(file.clone());
+        let id = managed_id(130, 4);
+        let want = hdr.read_managed_object(&file, &id, 8);
+        assert_eq!(hdr.read_managed_object_in(&storage, &id, 8), want);
+    }
+
+    /// A huge object found through the huge-object B-tree reads the
+    /// B-tree through Storage: the same result (here an error, there is no
+    /// B-tree at that address) as from the slice.
+    #[test]
+    fn huge_object_btree_reads_through_storage() {
+        use crate::storage::CountingStorage;
+        let (file, _) = build_simple_heap(8, 8);
+        let mut hdr = FractalHeapHeader::parse(&file, 0, 8, 8).unwrap();
+        hdr.huge_btree_address = 700;
+        let id = [0x10, 1, 0, 0, 0, 0, 0];
+        let want = hdr.read_managed_object(&file, &id, 8);
+        assert!(want.is_err());
+        let storage = CountingStorage::new(file);
+        assert_eq!(hdr.read_managed_object_in(&storage, &id, 8), want);
+    }
+
+    /// A header with an I/O filter pipeline (read in a second, longer
+    /// window) parses identically through a `read_at`-only storage, for
+    /// every truncation.
+    #[test]
+    fn filtered_header_parses_identically_through_storage() {
+        use crate::storage::CountingStorage;
+        let (simple, header_end) = build_simple_heap(8, 8);
+        let pipeline = [2u8, 1, 1, 0, 0, 0, 1, 0, 6, 0, 0, 0]; // deflate, level 6
+        let mut header = simple[..header_end - 4].to_vec();
+        header[7..9].copy_from_slice(&(pipeline.len() as u16).to_le_bytes());
+        header.extend_from_slice(&100u64.to_le_bytes()); // root block's stored size
+        header.extend_from_slice(&0u32.to_le_bytes()); // its filter mask
+        header.extend_from_slice(&pipeline);
+        let sum = crate::checksum::jenkins_lookup3(&header);
+        header.extend_from_slice(&sum.to_le_bytes());
+        let mut file = header.clone();
+        file.resize(256, 0);
+        let hdr = FractalHeapHeader::parse(&file, 0, 8, 8).unwrap();
+        assert!(hdr.filter_pipeline.is_some());
+        for cut in 0..=file.len() {
+            let f = &file[..cut];
+            let storage = CountingStorage::new(f.to_vec());
+            assert_eq!(
+                format!("{:?}", FractalHeapHeader::parse_in(&storage, 0, 8, 8)),
+                format!("{:?}", FractalHeapHeader::parse(f, 0, 8, 8)),
+                "cut {cut}"
+            );
+        }
+    }
 }

crates/clawhdf5-format/src/gather.rs

@@ -13,6 +13,7 @@ use alloc::{vec, vec::Vec};
 use crate::data_read::NativeElement;
 use crate::error::FormatError;
 use crate::selection::Selection;
+use crate::storage::{ExtentBytes, ExtentReq, Storage, raw_batches};
 
 /// Row-major element strides of `dims` (the last dimension has stride 1).
 fn strides(dims: &[u64]) -> Vec<u64> {
@@ -261,6 +262,259 @@ pub(crate) fn gather<T: NativeElement>(
     Ok(out)
 }
 
+/// Largest gap between two of a selection's runs that [`gather_storage`]
+/// reads through rather than asking for the runs separately: skipping a
+/// few KiB costs a remote backend far less than another request (and a
+/// local one less than another call and allocation).
+pub(crate) const GATHER_GAP_BYTES: usize = 4 << 10;
+
+/// Largest single read [`gather_storage`] makes of a selection's runs: runs
+/// are merged into reads up to this size, and a longer run is split.
+pub(crate) const GATHER_SPAN_BYTES: usize = 8 << 20;
+
+/// Call `emit(first_element, element_count)` for each run of a validated
+/// hyperslab or point selection (in output order; see [`hyperslab_runs`]),
+/// or the error for a hyperslab of the wrong rank or a point outside `dims`
+/// (runs before that point have been emitted).
+fn selection_runs(
+    dims: &[u64],
+    selection: &Selection,
+    emit: &mut dyn FnMut(u64, u64),
+) -> Result<(), FormatError> {
+    match selection {
+        Selection::Hyperslab {
+            start,
+            stride,
+            count,
+            block,
+        } => {
+            let rank = dims.len();
+            if [start.len(), stride.len(), count.len(), block.len()] != [rank; 4] {
+                return Err(FormatError::SelectionOutOfBounds(
+                    "hyperslab rank does not match dataset rank".into(),
+                ));
+            }
+            hyperslab_runs(dims, start, stride, count, block, emit);
+        }
+        Selection::Points(points) => {
+            let strides = strides(dims);
+            let mut coalesce = Coalesce {
+                start: 0,
+                len: 0,
+                emit,
+            };
+            for p in points {
+                if p.len() != dims.len() || p.iter().zip(dims).any(|(c, n)| c >= n) {
+                    return Err(FormatError::SelectionOutOfBounds(
+                        "selection addresses elements outside the dataset".into(),
+                    ));
+                }
+                let at = p
+                    .iter()
+                    .zip(&strides)
+                    .fold(0u64, |acc, (c, s)| acc.wrapping_add(c.wrapping_mul(*s)));
+                coalesce.push(at, 1);
+            }
+            coalesce.flush();
+        }
+        Selection::None | Selection::All => {}
+    }
+    Ok(())
+}
+
+/// One read of [`gather_storage`]: bytes `[start, end)` of the dataset,
+/// which hold the output's bytes up to `out_end` (from where the previous
+/// span's end left off).
+#[derive(Clone, Copy)]
+struct Span {
+    start: usize,
+    end: usize,
+    out_end: usize,
+}
+
+/// [`gather`] of bytes (`T = u8`) from a dataset that is not in memory: the
+/// dataset's `src_len` bytes start at `base` in `file`, which must hold all
+/// of them (the caller checks). Same checks and errors as [`gather`].
+///
+/// The selection's runs are walked twice. The first walk checks them and
+/// plans the reads: runs in increasing order with at most
+/// [`GATHER_GAP_BYTES`] between them are read as one span (the gap is read
+/// and dropped), up to [`GATHER_SPAN_BYTES`] per span. So a strided
+/// selection is a few large reads, not one per element, and nothing is
+/// allocated per run. The spans are fetched batch by batch (one
+/// [`Storage::read_ranges`] call per [`crate::storage::RAW_BATCH_BYTES`])
+/// while the second walk copies each run out of its span.
+pub(crate) fn gather_storage<S: Storage + ?Sized>(
+    file: &S,
+    base: u64,
+    src_len: usize,
+    dims: &[u64],
+    elem_size: usize,
+    selection: &Selection,
+) -> Result<Vec<u8>, FormatError> {
+    if elem_size == 0 {
+        return Err(FormatError::DataSizeMismatch {
+            expected: 1,
+            actual: elem_size,
+        });
+    }
+    let n_elements = match selection {
+        Selection::None => 0,
+        Selection::Hyperslab { count, block, .. } => count
+            .iter()
+            .zip(block)
+            .try_fold(1u64, |acc, (&c, &b)| acc.checked_mul(c.checked_mul(b)?))
+            .ok_or_else(|| FormatError::Overflow("hyperslab count x block overflows".into()))?,
+        Selection::Points(points) => points.len() as u64,
+        Selection::All => {
+            return Err(FormatError::SelectionOutOfBounds(
+                "gather does not take Selection::All".into(),
+            ));
+        }
+    };
+    let out_bytes = crate::chunked_read::checked_byte_len(n_elements, elem_size)?;
+    let outside = || {
+        FormatError::SelectionOutOfBounds("selection addresses elements outside the dataset".into())
+    };
+
+    // First walk: check every run and plan the spans.
+    let mut spans: Vec<Span> = Vec::new();
+    let mut total = 0usize;
+    let mut failed = false;
+    selection_runs(dims, selection, &mut |first: u64, n: u64| {
+        if failed {
+            return;
+        }
+        let range = usize::try_from(first)
+            .ok()
+            .and_then(|f| f.checked_mul(elem_size))
+            .zip(
+                usize::try_from(n)
+                    .ok()
+                    .and_then(|n| n.checked_mul(elem_size)),
+            )
+            .and_then(|(at, len)| Some((at, len, at.checked_add(len)?)));
+        let Some((mut at, mut len)) = range
+            .filter(|&(_, len, end)| end <= src_len && len <= out_bytes - total)
+            .map(|(at, len, _)| (at, len))
+        else {
+            failed = true;
+            return;
+        };
+        while len > 0 {
+            let room = match spans.last_mut() {
+                Some(s)
+                    if at >= s.end
+                        && at - s.end <= GATHER_GAP_BYTES
+                        && at - s.start < GATHER_SPAN_BYTES =>
+                {
+                    let take = len.min(GATHER_SPAN_BYTES - (at - s.start));
+                    s.end = at + take;
+                    s.out_end += take;
+                    take
+                }
+                _ => {
+                    let take = len.min(GATHER_SPAN_BYTES);
+                    spans.push(Span {
+                        start: at,
+                        end: at + take,
+                        out_end: total + take,
+                    });
+                    take
+                }
+            };
+            total += room;
+            at += room;
+            len -= room;
+        }
+    })?;
+    if failed || total != out_bytes {
+        return Err(outside());
+    }
+
+    // The spans' reads, and the batches they are fetched in.
+    let reqs: Vec<ExtentReq> = spans
+        .iter()
+        .map(|s| ExtentReq {
+            addr: base + s.start as u64,
+            len: s.end - s.start,
+            fetch: Some(s.end - s.start),
+        })
+        .collect();
+    let batches = raw_batches(reqs.len(), false, |i| reqs[i].len);
+
+    // Second walk: copy each run out of its span, fetching each batch of
+    // spans when the walk reaches it (and dropping the previous one).
+    let mut out = crate::bulk_alloc::vec_for_bulk(out_bytes);
+    let mut span = 0usize;
+    let mut batch = 0usize;
+    let mut fetched: Option<ExtentBytes<'_>> = None;
+    let mut error: Option<FormatError> = None;
+    selection_runs(dims, selection, &mut |first: u64, n: u64| {
+        if error.is_some() {
+            return;
+        }
+        // Checked by the first walk (these cannot saturate or wrap).
+        let mut at = crate::addr::saturating_usize(first).wrapping_mul(elem_size);
+        let mut len = crate::addr::saturating_usize(n).wrapping_mul(elem_size);
+        while len > 0 {
+            while spans.get(span).is_some_and(|s| s.out_end <= out.len()) {
+                span += 1;
+            }
+            if fetched.is_none() || span >= batches[batch].end {
+                fetched = None;
+                while batches.get(batch).is_some_and(|b| span >= b.end) {
+                    batch += 1;
+                }
+                let (Some(b), Some(_)) = (batches.get(batch).cloned(), spans.get(span)) else {
+                    // The second walk emitted more than the first.
+                    error = Some(outside());
+                    return;
+                };
+                match ExtentBytes::fetch(file, &reqs[b.clone()], b.start) {
+                    Ok(f) => fetched = Some(f),
+                    Err(e) => {
+                        error = Some(e);
+                        return;
+                    }
+                }
+            }
+            let s = spans[span];
+            let take = len.min(s.out_end - out.len());
+            let bytes = match fetched
+                .as_ref()
+                .map(|f| f.get(span, &reqs[span]))
+                .unwrap_or_else(|| Err(outside()))
+            {
+                Ok(b) => b,
+                Err(e) => {
+                    error = Some(e);
+                    return;
+                }
+            };
+            match at
+                .checked_sub(s.start)
+                .and_then(|o| bytes.get(o..o.checked_add(take)?))
+            {
+                Some(b) => out.extend_from_slice(b),
+                None => {
+                    error = Some(outside());
+                    return;
+                }
+            }
+            at += take;
+            len -= take;
+        }
+    })?;
+    if let Some(e) = error {
+        return Err(e);
+    }
+    if out.len() != out_bytes {
+        return Err(outside());
+    }
+    Ok(out)
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

crates/clawhdf5-format/src/global_heap.rs

@@ -1,9 +1,12 @@
 //! HDF5 Global Heap collection parsing.
 
 #[cfg(not(feature = "std"))]
-use alloc::{format, string::String, vec::Vec};
+use alloc::{borrow::Cow, format, string::String, vec::Vec};
+#[cfg(feature = "std")]
+use std::borrow::Cow;
 
 use crate::error::FormatError;
+use crate::storage::{Storage, len_usize, read_exact_at};
 
 /// Magic signature for global heap collections.
 const GCOL_SIGNATURE: [u8; 4] = *b"GCOL";
@@ -28,19 +31,20 @@ pub struct GlobalHeapObject {
     pub data: Vec<u8>,
 }
 
-fn ensure_len(data: &[u8], offset: usize, needed: usize) -> Result<(), FormatError> {
+/// Checks that `[offset, offset + needed)` ends by `data_len`.
+fn ensure_len(data_len: usize, offset: usize, needed: usize) -> Result<(), FormatError> {
     match offset.checked_add(needed) {
-        Some(end) if end <= data.len() => Ok(()),
+        Some(end) if end <= data_len => Ok(()),
         _ => Err(FormatError::UnexpectedEof {
             expected: offset.saturating_add(needed),
-            available: data.len(),
+            available: data_len,
         }),
     }
 }
 
 fn read_length(data: &[u8], offset: usize, length_size: u8) -> Result<u64, FormatError> {
     let s = length_size as usize;
-    ensure_len(data, offset, s)?;
+    ensure_len(data.len(), offset, s)?;
     let slice = &data[offset..offset + s];
     Ok(match length_size {
         2 => u16::from_le_bytes([slice[0], slice[1]]) as u64,
@@ -95,7 +99,17 @@ impl GlobalHeapCollection {
         offset: usize,
         length_size: u8,
     ) -> Result<GlobalHeapCollection, FormatError> {
-        let index = Self::parse_index(file_data, offset, length_size)?;
+        Self::parse_in(file_data, offset as u64, length_size)
+    }
+
+    /// [`Self::parse`] over any [`Storage`]: one read of the header, one of
+    /// the collection.
+    pub fn parse_in<S: Storage + ?Sized>(
+        file: &S,
+        offset: u64,
+        length_size: u8,
+    ) -> Result<GlobalHeapCollection, FormatError> {
+        let (bytes, base, index) = Self::read_collection(file, offset, length_size)?;
         Ok(GlobalHeapCollection {
             collection_size: index.collection_size,
             objects: index
@@ -104,7 +118,7 @@ impl GlobalHeapCollection {
                 .map(|o| GlobalHeapObject {
                     index: o.index,
                     reference_count: o.reference_count,
-                    data: file_data[o.offset..o.offset + o.size].to_vec(),
+                    data: bytes[o.offset - base..o.offset - base + o.size].to_vec(),
                 })
                 .collect(),
         })
@@ -122,43 +136,72 @@ impl GlobalHeapCollection {
         offset: usize,
         length_size: u8,
     ) -> Result<GlobalHeapIndex, FormatError> {
+        Self::parse_index_in(file_data, offset as u64, length_size)
+    }
+
+    /// [`Self::parse_index`] over any [`Storage`]: one read of the header,
+    /// one of the collection. The object offsets are file offsets.
+    pub fn parse_index_in<S: Storage + ?Sized>(
+        file: &S,
+        offset: u64,
+        length_size: u8,
+    ) -> Result<GlobalHeapIndex, FormatError> {
+        Ok(Self::read_collection(file, offset, length_size)?.2)
+    }
+
+    /// Read the collection at `offset` and index its objects: the
+    /// collection's bytes, its offset as a `usize`, and the index (with
+    /// file offsets).
+    pub(crate) fn read_collection<S: Storage + ?Sized>(
+        file: &S,
+        offset: u64,
+        length_size: u8,
+    ) -> Result<(Cow<'_, [u8]>, usize, GlobalHeapIndex), FormatError> {
+        let file_len = len_usize(file);
         // signature(4) + version(1) + reserved(3) + collection_size(length_size),
         // padded to a multiple of 8 as libhdf5 lays it out (`H5HG_SIZEOF_HDR`).
         // With 8-byte lengths the padding is 0; with 4-byte lengths it is 4,
         // and reading without it put every object 4 bytes early.
         let header_size = pad8(8 + length_size as usize);
-        ensure_len(file_data, offset, header_size)?;
+        let header = read_exact_at(file, offset, header_size)?;
+        let offset = usize::try_from(offset).map_err(|_| FormatError::UnexpectedEof {
+            expected: usize::MAX,
+            available: file_len,
+        })?;
 
-        if file_data[offset..offset + 4] != GCOL_SIGNATURE {
+        if header[..4] != GCOL_SIGNATURE {
             return Err(FormatError::InvalidGlobalHeapSignature);
         }
 
-        let version = file_data[offset + 4];
+        let version = header[4];
         if version != 1 {
             return Err(FormatError::InvalidGlobalHeapVersion(version));
         }
 
-        let collection_size = read_length(file_data, offset + 8, length_size)?;
+        let collection_size = read_length(&header, 8, length_size)?;
         let collection_end = usize::try_from(collection_size)
             .ok()
             .and_then(|size| offset.checked_add(size))
             .ok_or(FormatError::UnexpectedEof {
                 expected: usize::MAX,
-                available: file_data.len(),
+                available: file_len,
             })?;
-        if collection_end > file_data.len() {
+        if collection_end > file_len {
             return Err(FormatError::UnexpectedEof {
                 expected: collection_end,
-                available: file_data.len(),
+                available: file_len,
             });
         }
+        let collection = read_exact_at(file, offset as u64, collection_end - offset)?;
+        // Positions below are file offsets; `file_data(p)` is the byte at `p`.
+        let file_data = |p: usize| collection[p - offset];
 
         let mut pos = offset + header_size;
         let mut objects = Vec::new();
 
         // Parse objects until we hit index 0 (free space) or run out of space
         while pos + 2 <= collection_end {
-            let object_index = u16::from_le_bytes([file_data[pos], file_data[pos + 1]]);
+            let object_index = u16::from_le_bytes([file_data(pos), file_data(pos + 1)]);
 
             if object_index == 0 {
                 // Free space marker — done
@@ -168,10 +211,11 @@ impl GlobalHeapCollection {
             // object_index(2) + reference_count(2) + reserved(4) +
             // object_size(length_size), padded to 8 (`H5HG_SIZEOF_OBJHDR`).
             let obj_header_size = pad8(8 + length_size as usize);
-            ensure_len(&file_data[..collection_end], pos, obj_header_size)?;
+            ensure_len(collection_end, pos, obj_header_size)?;
 
-            let reference_count = u16::from_le_bytes([file_data[pos + 2], file_data[pos + 3]]);
+            let reference_count = u16::from_le_bytes([file_data(pos + 2), file_data(pos + 3)]);
-            let object_size = usize::try_from(read_length(file_data, pos + 8, length_size)?)
+            let object_size =
+                usize::try_from(read_length(&collection[pos - offset..], 8, length_size)?)
                     .map_err(|_| FormatError::Overflow("global heap object size".into()))?;
 
             pos += obj_header_size;
@@ -197,10 +241,11 @@ impl GlobalHeapCollection {
             pos = pos.saturating_add(pad8(object_size));
         }
 
-        Ok(GlobalHeapIndex {
+        let index = GlobalHeapIndex {
             collection_size,
             objects,
-        })
+        };
+        Ok((collection, offset, index))
     }
 
     /// Get an object by its index.
@@ -226,7 +271,7 @@ mod tests {
         let mut obj_size_total = 0usize;
         for (_, _, data) in objects {
             let obj_header = pad8(8 + ls);
-            obj_size_total += obj_header + pad8(data.len());
+            obj_size_total += obj_header + pad8(<[u8]>::len(data));
         }
         // Free space marker (2 bytes for index 0)
         obj_size_total += 2;
@@ -251,15 +296,17 @@ mod tests {
             buf.extend_from_slice(&ref_count.to_le_bytes());
             buf.extend_from_slice(&[0u8; 4]); // reserved
             match length_size {
-                4 => buf.extend_from_slice(&(data.len() as u32).to_le_bytes()),
+                // `<[u8]>::len`: with `Storage` in scope `data.len()` on a
-                8 => buf.extend_from_slice(&(data.len() as u64).to_le_bytes()),
+                // `&&[u8]` resolves to `Storage::len` (a `u64`).
+                4 => buf.extend_from_slice(&(<[u8]>::len(data) as u32).to_le_bytes()),
+                8 => buf.extend_from_slice(&(<[u8]>::len(data) as u64).to_le_bytes()),
                 _ => panic!("unsupported"),
             }
             buf.resize(buf.len() + (pad8(8 + ls) - (8 + ls)), 0);
             buf.extend_from_slice(data);
             // Pad to 8 bytes
-            let padded = pad8(data.len());
+            let padded = pad8(<[u8]>::len(data));
-            buf.resize(buf.len() + (padded - data.len()), 0);
+            buf.resize(buf.len() + (padded - <[u8]>::len(data)), 0);
         }
 
         // Free space marker
@@ -327,4 +374,44 @@ mod tests {
         assert_eq!(coll.objects.len(), 1);
         assert_eq!(coll.objects[0].data, b"test");
     }
+
+    /// Collections, and every truncation of them, index and parse
+    /// identically through a `read_at`-only storage: two reads each.
+    #[test]
+    fn storage_parse_matches_slice_parse() {
+        use crate::storage::CountingStorage;
+        let objs: &[(u16, u16, &[u8])] = &[(1, 1, b"hello"), (2, 3, b"a longer object")];
+        for ls in [4u8, 8] {
+            let coll = build_collection(objs, ls);
+            let mut corrupt = coll.clone();
+            corrupt[8] = 200; // collection size past the end of the file
+            let mut overrun = coll.clone();
+            let size_at = pad8(8 + ls as usize) + 8;
+            overrun[size_at] = 250; // first object runs past the collection
+            for full in [coll, corrupt, overrun] {
+                for at in [0usize, 5] {
+                    for cut in 0..=full.len() {
+                        let mut f = vec![0u8; at];
+                        f.extend_from_slice(&full[..cut]);
+                        let storage = CountingStorage::new(f.clone());
+                        let want = GlobalHeapCollection::parse(&f, at, ls);
+                        let got = GlobalHeapCollection::parse_in(&storage, at as u64, ls);
+                        assert_eq!(format!("{got:?}"), format!("{want:?}"));
+                        let want = GlobalHeapCollection::parse_index(&f, at, ls);
+                        let got = GlobalHeapCollection::parse_index_in(&storage, at as u64, ls);
+                        assert_eq!(format!("{got:?}"), format!("{want:?}"));
+                    }
+                }
+            }
+        }
+        let storage = CountingStorage::new(build_collection(objs, 8));
+        assert_eq!(
+            GlobalHeapCollection::parse_in(&storage, 0, 8)
+                .unwrap()
+                .objects
+                .len(),
+            2
+        );
+        assert_eq!(storage.reads(), 2);
+    }
 }

crates/clawhdf5-format/src/group_v1.rs

@@ -3,11 +3,13 @@
 #[cfg(not(feature = "std"))]
 use alloc::{string::String, vec::Vec};
 
-use crate::btree_v1::collect_symbol_table_nodes;
+use crate::addr::checked_addr;
+use crate::btree_v1::{BTreeV1Node, collect_symbol_table_nodes_in};
 use crate::error::FormatError;
 use crate::local_heap::LocalHeap;
 use crate::message_type::MessageType;
 use crate::object_header::ObjectHeader;
+use crate::storage::Storage;
 use crate::symbol_table::{SymbolTableMessage, SymbolTableNode};
 
 /// A resolved group entry (child name + object header address).
@@ -21,50 +23,212 @@ pub struct GroupEntry {
     pub cache_type: u32,
 }
 
-/// Given a SymbolTableMessage, resolve all group children.
+/// Given a SymbolTableMessage, resolve all group children: the group's
+/// listing.
+///
+/// An entry with an empty name fails the listing with
+/// [`FormatError::InvalidLinkName`], as it fails libhdf5's link iteration
+/// (`H5G__ent_to_link`: "invalid link name"). Looking a name up
+/// ([`resolve_path`], and the path resolution in
+/// [`crate::group_v2::resolve_path_any`]) still works in such a group, as it
+/// does in libhdf5.
 pub fn resolve_v1_group_entries(
     file_data: &[u8],
     sym_table_msg: &SymbolTableMessage,
     offset_size: u8,
     length_size: u8,
+) -> Result<Vec<GroupEntry>, FormatError> {
+    resolve_v1_group_entries_in(file_data, sym_table_msg, offset_size, length_size)
+}
+
+/// [`resolve_v1_group_entries`] over any [`Storage`].
+pub fn resolve_v1_group_entries_in<S: Storage + ?Sized>(
+    file_data: &S,
+    sym_table_msg: &SymbolTableMessage,
+    offset_size: u8,
+    length_size: u8,
+) -> Result<Vec<GroupEntry>, FormatError> {
+    let entries = v1_group_entries(file_data, sym_table_msg, offset_size, length_size, true)?;
+    if entries.iter().any(|e| e.name.is_empty()) {
+        return Err(FormatError::InvalidLinkName);
+    }
+    Ok(entries)
+}
+
+/// Every entry of a v1 group, empty names included — for looking a name up,
+/// which never matches an empty name.
+///
+/// With `hint_headers` (a listing, whose children are usually opened
+/// next), each entry's object header is hinted (see
+/// [`Storage::hint`]) as soon as its symbol table node is read.
+pub(crate) fn v1_group_entries<S: Storage + ?Sized>(
+    file_data: &S,
+    sym_table_msg: &SymbolTableMessage,
+    offset_size: u8,
+    length_size: u8,
+    hint_headers: bool,
 ) -> Result<Vec<GroupEntry>, FormatError> {
     // Parse local heap
-    let heap = LocalHeap::parse(
+    let heap = LocalHeap::parse_in(
         file_data,
-        sym_table_msg.local_heap_address as usize,
+        checked_addr(sym_table_msg.local_heap_address)?,
         offset_size,
         length_size,
     )?;
 
     // Collect all SNOD addresses from B-tree
-    let snod_addrs = collect_symbol_table_nodes(
+    let snod_addrs = collect_symbol_table_nodes_in(
         file_data,
         sym_table_msg.btree_address,
         offset_size,
         length_size,
     )?;
 
+    // The names are read one by one from the heap's data segment; read
+    // (up to 1 MiB of) it first, so a storage that records what it lacks
+    // asks for it at once (see `storage::touch`).
+    if !snod_addrs.is_empty() {
+        let len = usize::try_from(heap.data_segment_size).map_or(1 << 20, |n| n.min(1 << 20));
+        crate::storage::touch(file_data, heap.data_segment_address, len);
+    }
+
     let mut entries = Vec::new();
     let mut heap_checked = false;
+    // After the first node that fails, the others are only read (as
+    // `storage::touch` does); that error is returned.
+    let mut failed = None;
     for snod_addr in snod_addrs {
-        let snod = SymbolTableNode::parse(file_data, snod_addr as usize, offset_size)?;
+        let snod = checked_addr(snod_addr)
+            .and_then(|a| SymbolTableNode::parse_in(file_data, a, offset_size));
+        if hint_headers && let Ok(snod) = &snod {
             for entry in &snod.entries {
-            // Like libhdf5, look at the heap's free list only once a name is
+                if entry.object_header_address != u64::MAX {
-            // needed: an empty group with a damaged heap still lists.
+                    file_data.hint(
+                        entry.object_header_address,
+                        crate::object_header::OBJECT_HEADER_HINT_LEN,
+                    );
+                }
+            }
+        }
+        if failed.is_some() {
+            continue;
+        }
+        let node = || -> Result<(), FormatError> {
+            let snod = snod?;
+            for entry in &snod.entries {
+                // Like libhdf5, look at the heap's free list only once a name
+                // is needed: an empty group with a damaged heap still lists.
                 if !heap_checked {
-                heap.validate_free_list(file_data, length_size)?;
+                    heap.validate_free_list_in(file_data, length_size)?;
                     heap_checked = true;
                 }
-            let name = heap.read_string(file_data, entry.link_name_offset)?;
+                let name = heap.read_string_in(file_data, entry.link_name_offset)?;
                 entries.push(GroupEntry {
                     name,
                     object_header_address: entry.object_header_address,
                     cache_type: entry.cache_type,
                 });
             }
+            Ok(())
+        };
+        if let Err(e) = node() {
+            failed = Some(e);
+        }
     }
 
-    Ok(entries)
+    match failed {
+        Some(e) => Err(e),
+        None => Ok(entries),
+    }
+}
+
+/// The entry called `name` in a v1 group, looked up as libhdf5 looks it up
+/// (`H5G__stab_lookup`: `H5B_find` down the group's B-tree, then
+/// `H5G__node_found` in one symbol table node) instead of by reading every
+/// entry: at each node, the child whose key interval holds the name
+/// (left key < name <= right key, keys being names in the local heap,
+/// compared bytewise as `strcmp` does) is found by binary search. Reads
+/// O(depth) nodes and names, where listing reads the whole group.
+///
+/// `Ok(None)` when the search does not lead to the name. In a group whose
+/// B-tree is out of name order (damaged, or made by hand) that does not
+/// prove it absent, so callers then fall back to reading every entry;
+/// libhdf5 would report it missing.
+pub(crate) fn find_v1_entry<S: Storage + ?Sized>(
+    file_data: &S,
+    sym_table_msg: &SymbolTableMessage,
+    name: &str,
+    offset_size: u8,
+    length_size: u8,
+) -> Result<Option<GroupEntry>, FormatError> {
+    let heap = LocalHeap::parse_in(
+        file_data,
+        checked_addr(sym_table_msg.local_heap_address)?,
+        offset_size,
+        length_size,
+    )?;
+    // The keys and names are read from the heap's data segment one at a
+    // time: hint it (see `Storage::hint`).
+    file_data.hint(
+        heap.data_segment_address,
+        usize::try_from(heap.data_segment_size).map_or(1 << 20, |n| n.min(1 << 20)),
+    );
+    // As in listing: the heap's free list is checked before a name is used.
+    let mut heap_checked = false;
+    let mut name_at = |offset: u64| -> Result<String, FormatError> {
+        if !heap_checked {
+            heap.validate_free_list_in(file_data, length_size)?;
+            heap_checked = true;
+        }
+        heap.read_string_in(file_data, offset)
+    };
+    let want = name.as_bytes();
+    let mut address = sym_table_msg.btree_address;
+    for _ in 0..=crate::btree_v1::MAX_BTREE_DEPTH {
+        let node =
+            BTreeV1Node::parse_in(file_data, checked_addr(address)?, offset_size, length_size)?;
+        if node.node_type != 0 {
+            return Err(FormatError::InvalidBTreeNodeType(node.node_type));
+        }
+        // H5B_find's binary search with H5G__node_cmp3: go left when the
+        // name sorts at or before the left key, right when after the right
+        // key; otherwise this child holds it.
+        let (mut lo, mut hi) = (0usize, node.children.len());
+        let mut child = None;
+        while lo < hi {
+            let i = lo + (hi - lo) / 2;
+            let (Some(&left), Some(&right)) = (node.keys.get(i), node.keys.get(i + 1)) else {
+                return Ok(None);
+            };
+            if want <= name_at(left)?.as_bytes() {
+                hi = i;
+            } else if want > name_at(right)?.as_bytes() {
+                lo = i + 1;
+            } else {
+                child = Some(node.children[i]);
+                break;
+            }
+        }
+        let Some(child) = child else {
+            return Ok(None);
+        };
+        if node.node_level > 0 {
+            address = child;
+            continue;
+        }
+        let snod = SymbolTableNode::parse_in(file_data, checked_addr(child)?, offset_size)?;
+        for entry in &snod.entries {
+            if name_at(entry.link_name_offset)?.as_bytes() == want {
+                return Ok(Some(GroupEntry {
+                    name: String::from(name),
+                    object_header_address: entry.object_header_address,
+                    cache_type: entry.cache_type,
+                }));
+            }
+        }
+        return Ok(None);
+    }
+    Err(FormatError::NestingDepthExceeded)
 }
 
 /// Symbol table cache type for a soft link: the scratch pad's first four bytes
@@ -79,6 +243,17 @@ pub fn find_v1_soft_link(
     name: &str,
     offset_size: u8,
     length_size: u8,
+) -> Result<Option<String>, FormatError> {
+    find_v1_soft_link_in(file_data, sym_table_msg, name, offset_size, length_size)
+}
+
+/// [`find_v1_soft_link`] over any [`Storage`].
+pub fn find_v1_soft_link_in<S: Storage + ?Sized>(
+    file_data: &S,
+    sym_table_msg: &SymbolTableMessage,
+    name: &str,
+    offset_size: u8,
+    length_size: u8,
 ) -> Result<Option<String>, FormatError> {
     let mut found = None;
     for_each_v1_soft_link(
@@ -101,6 +276,16 @@ pub fn v1_soft_links(
     sym_table_msg: &SymbolTableMessage,
     offset_size: u8,
     length_size: u8,
+) -> Result<Vec<(String, String)>, FormatError> {
+    v1_soft_links_in(file_data, sym_table_msg, offset_size, length_size)
+}
+
+/// [`v1_soft_links`] over any [`Storage`].
+pub fn v1_soft_links_in<S: Storage + ?Sized>(
+    file_data: &S,
+    sym_table_msg: &SymbolTableMessage,
+    offset_size: u8,
+    length_size: u8,
 ) -> Result<Vec<(String, String)>, FormatError> {
     let mut links = Vec::new();
     for_each_v1_soft_link(
@@ -119,21 +304,21 @@ pub fn v1_soft_links(
 
 /// Visit the soft links of a v1 group whose name passes `wanted`, with their
 /// target paths, until `visit` returns false.
-fn for_each_v1_soft_link(
+fn for_each_v1_soft_link<S: Storage + ?Sized>(
-    file_data: &[u8],
+    file_data: &S,
     sym_table_msg: &SymbolTableMessage,
     offset_size: u8,
     length_size: u8,
     wanted: impl Fn(&str) -> bool,
     mut visit: impl FnMut(&str, String) -> bool,
 ) -> Result<(), FormatError> {
-    let heap = LocalHeap::parse(
+    let heap = LocalHeap::parse_in(
         file_data,
-        sym_table_msg.local_heap_address as usize,
+        checked_addr(sym_table_msg.local_heap_address)?,
         offset_size,
         length_size,
     )?;
-    let snod_addrs = collect_symbol_table_nodes(
+    let snod_addrs = collect_symbol_table_nodes_in(
         file_data,
         sym_table_msg.btree_address,
         offset_size,
@@ -141,16 +326,16 @@ fn for_each_v1_soft_link(
     )?;
     let mut heap_checked = false;
     for snod_addr in snod_addrs {
-        let snod = SymbolTableNode::parse(file_data, snod_addr as usize, offset_size)?;
+        let snod = SymbolTableNode::parse_in(file_data, checked_addr(snod_addr)?, offset_size)?;
         for entry in &snod.entries {
             if entry.cache_type != CACHE_TYPE_SOFT_LINK {
                 continue;
             }
             if !heap_checked {
-                heap.validate_free_list(file_data, length_size)?;
+                heap.validate_free_list_in(file_data, length_size)?;
                 heap_checked = true;
             }
-            let name = heap.read_string(file_data, entry.link_name_offset)?;
+            let name = heap.read_string_in(file_data, entry.link_name_offset)?;
             if !wanted(&name) {
                 continue;
             }
@@ -160,7 +345,7 @@ fn for_each_v1_soft_link(
                 entry.scratch_pad[2],
                 entry.scratch_pad[3],
             ]);
-            let target = heap.read_string(file_data, u64::from(value_offset))?;
+            let target = heap.read_string_in(file_data, u64::from(value_offset))?;
             if !visit(&name, target) {
                 return Ok(());
             }
@@ -198,6 +383,17 @@ pub fn resolve_path(
     path: &str,
     offset_size: u8,
     length_size: u8,
+) -> Result<u64, FormatError> {
+    resolve_path_in(file_data, root_sym_table, path, offset_size, length_size)
+}
+
+/// [`resolve_path`] over any [`Storage`].
+pub fn resolve_path_in<S: Storage + ?Sized>(
+    file_data: &S,
+    root_sym_table: &SymbolTableMessage,
+    path: &str,
+    offset_size: u8,
+    length_size: u8,
 ) -> Result<u64, FormatError> {
     let components: Vec<&str> = path.split('/').filter(|s| !s.is_empty()).collect();
     if components.is_empty() {
@@ -207,8 +403,13 @@ pub fn resolve_path(
     let mut current_sym_table = root_sym_table.clone();
 
     for (i, component) in components.iter().enumerate() {
-        let entries =
+        let entries = v1_group_entries(
-            resolve_v1_group_entries(file_data, &current_sym_table, offset_size, length_size)?;
+            file_data,
+            &current_sym_table,
+            offset_size,
+            length_size,
+            false,
+        )?;
 
         let found = entries.iter().find(|e| e.name == *component);
         match found {
@@ -218,9 +419,9 @@ pub fn resolve_path(
                     return Ok(entry.object_header_address);
                 }
                 // Not last — must be a group, parse its object header to get symbol table
-                let obj_header = ObjectHeader::parse(
+                let obj_header = ObjectHeader::parse_in(
                     file_data,
-                    entry.object_header_address as usize,
+                    checked_addr(entry.object_header_address)?,
                     offset_size,
                     length_size,
                 )?;
@@ -425,6 +626,22 @@ mod tests {
         assert_eq!(entries[1].object_header_address, 0x2000);
     }
 
+    /// cve-2021-46244 `/BAG_root`: a symbol-table entry with an empty name.
+    /// libhdf5 fails the group's listing ("invalid link name"); a lookup of
+    /// the other names still works.
+    #[test]
+    fn empty_entry_name_fails_the_listing_not_a_lookup() {
+        let (file, msg) = build_synthetic_group(&[("", 0x1000, 0), ("elevation", 0x2000, 0)], 8, 8);
+        assert_eq!(
+            resolve_v1_group_entries(&file, &msg, 8, 8).unwrap_err(),
+            FormatError::InvalidLinkName
+        );
+        assert_eq!(
+            resolve_path(&file, &msg, "elevation", 8, 8).unwrap(),
+            0x2000
+        );
+    }
+
     #[test]
     fn resolve_path_single_level() {
         let (file, msg) =

crates/clawhdf5-format/src/group_v2.rs

@@ -6,7 +6,14 @@
 #[cfg(not(feature = "std"))]
 use alloc::{string::String, vec::Vec};
 
-use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records};
+#[cfg(not(feature = "std"))]
+use alloc::collections::BTreeSet;
+#[cfg(feature = "std")]
+use std::collections::BTreeSet;
+
+use crate::addr::checked_addr;
+use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records_in, find_btree_v2_records_in};
+use crate::checksum::jenkins_lookup3;
 use crate::error::FormatError;
 use crate::fractal_heap::FractalHeapHeader;
 use crate::group_v1::{self, GroupEntry};
@@ -14,6 +21,7 @@ use crate::link_info::LinkInfoMessage;
 use crate::link_message::{LinkMessage, LinkTarget};
 use crate::message_type::MessageType;
 use crate::object_header::ObjectHeader;
+use crate::storage::Storage;
 use crate::superblock::Superblock;
 use crate::symbol_table::SymbolTableMessage;
 
@@ -25,6 +33,16 @@ pub fn resolve_v2_group_entries(
     object_header: &ObjectHeader,
     offset_size: u8,
     length_size: u8,
+) -> Result<Vec<GroupEntry>, FormatError> {
+    resolve_v2_group_entries_in(file_data, object_header, offset_size, length_size)
+}
+
+/// [`resolve_v2_group_entries`] over any [`Storage`].
+pub fn resolve_v2_group_entries_in<S: Storage + ?Sized>(
+    file_data: &S,
+    object_header: &ObjectHeader,
+    offset_size: u8,
+    length_size: u8,
 ) -> Result<Vec<GroupEntry>, FormatError> {
     // Look for Link Info message to determine storage type
     let link_info = find_link_info(object_header, offset_size)?;
@@ -83,25 +101,66 @@ fn resolve_compact_entries(
     Ok(entries)
 }
 
-/// Visit every link in dense storage (fractal heap + B-tree v2 name index).
+/// What a version-2 B-tree header takes with 8-byte offsets and lengths
-fn for_each_dense_link(
+/// (22 bytes of fields, the root node's address and record count, and the
-    file_data: &[u8],
+/// checksum), rounded up: hinted before one is read.
+const BTREE_V2_HEADER_HINT_LEN: usize = 64;
+
+/// The fractal heap of a dense group. The name index's header and the
+/// heap's root block are read next, whatever the lookup: they are hinted
+/// (see [`Storage::hint`]) so that a storage fetching between attempts
+/// gets them in the same round trip as the heap's header.
+fn dense_heap<S: Storage + ?Sized>(
+    file_data: &S,
     link_info: &LinkInfoMessage,
     fh_addr: u64,
     offset_size: u8,
     length_size: u8,
+) -> Result<FractalHeapHeader, FormatError> {
+    if let Some(btree_addr) = link_info.btree_name_index_address {
+        file_data.hint(btree_addr, BTREE_V2_HEADER_HINT_LEN);
+    }
+    let fh =
+        FractalHeapHeader::parse_in(file_data, checked_addr(fh_addr)?, offset_size, length_size)?;
+    fh.hint_root_block(file_data);
+    Ok(fh)
+}
+
+/// Visit every link in dense storage (fractal heap + B-tree v2 name index).
+///
+/// With `hint_headers` (a listing, whose children are usually opened
+/// next), the object header of every hard link is hinted (see
+/// [`Storage::hint`]) as soon as the link is read, even after a failure.
+fn for_each_dense_link<S: Storage + ?Sized>(
+    file_data: &S,
+    link_info: &LinkInfoMessage,
+    fh_addr: u64,
+    offset_size: u8,
+    length_size: u8,
+    hint_headers: bool,
     mut visit: impl FnMut(LinkMessage),
 ) -> Result<(), FormatError> {
-    // Parse fractal heap
+    let fh = dense_heap(file_data, link_info, fh_addr, offset_size, length_size)?;
-    let fh = FractalHeapHeader::parse(file_data, fh_addr as usize, offset_size, length_size)?;
+    if hint_headers {
+        // Every link is read: so is every block of the heap.
+        fh.hint_managed_blocks(file_data);
+    }
 
     // Parse B-tree v2 for name index
     let btree_addr = link_info
         .btree_name_index_address
         .ok_or_else(|| FormatError::PathNotFound(String::from("no B-tree v2 name index")))?;
-    let btree_hdr = BTreeV2Header::parse(file_data, btree_addr as usize, offset_size, length_size)?;
+    let btree_hdr = BTreeV2Header::parse_in(
-    let records = collect_btree_v2_records(file_data, &btree_hdr, offset_size, length_size)?;
+        file_data,
+        checked_addr(btree_addr)?,
+        offset_size,
+        length_size,
+    )?;
+    let records = collect_btree_v2_records_in(file_data, &btree_hdr, offset_size, length_size)?;
 
+    // After the first link that fails, the others are only read, not
+    // visited (a touch, see `storage::touch`); that error is returned.
+    let mut failed = None;
     for record in &records {
         // For type 5 (name index): hash(4) + heap_id(heap_id_length)
         // For type 6 (creation order): creation_order(8) + heap_id(heap_id_length)
@@ -117,17 +176,41 @@ fn for_each_dense_link(
         let id_bytes = &record.data[id_offset..id_offset + fh.heap_id_length as usize];
 
         // Read managed object from fractal heap
-        let link_data = fh.read_managed_object(file_data, id_bytes, offset_size)?;
+        let link = fh
-        if let Some(link) = parse_link(&link_data, offset_size)? {
+            .read_managed_object_in(file_data, id_bytes, offset_size)
-            visit(link);
+            .and_then(|d| parse_link(&d, offset_size));
+        if hint_headers
+            && let Ok(Some(LinkMessage {
+                link_target:
+                    LinkTarget::Hard {
+                        object_header_address,
+                    },
+                ..
+            })) = &link
+        {
+            file_data.hint(
+                *object_header_address,
+                crate::object_header::OBJECT_HEADER_HINT_LEN,
+            );
+        }
+        if failed.is_some() {
+            continue;
+        }
+        match link {
+            Ok(Some(link)) => visit(link),
+            Ok(None) => {}
+            Err(e) => failed = Some(e),
         }
     }
-    Ok(())
+    match failed {
+        Some(e) => Err(e),
+        None => Ok(()),
+    }
 }
 
 /// Resolve entries from dense storage (fractal heap + B-tree v2).
-fn resolve_dense_entries(
+fn resolve_dense_entries<S: Storage + ?Sized>(
-    file_data: &[u8],
+    file_data: &S,
     link_info: &LinkInfoMessage,
     fh_addr: u64,
     offset_size: u8,
@@ -140,6 +223,7 @@ fn resolve_dense_entries(
         fh_addr,
         offset_size,
         length_size,
+        true,
         |link| {
             if let LinkTarget::Hard {
                 object_header_address,
@@ -156,17 +240,16 @@ fn resolve_dense_entries(
     Ok(entries)
 }
 
-/// The soft or external link called `name` in this group, if there is one.
+/// The soft link called `name` in a v1 (symbol table) group, if there is
-/// Hard links are what `resolve_group_entries` returns; this is consulted only
+/// one. Hard links are what `resolve_group_entries` returns; this is
-/// when a path component isn't among them.
+/// consulted only when a path component isn't among them.
-fn find_symbolic_link(
+fn find_v1_symbolic_link<S: Storage + ?Sized>(
-    file_data: &[u8],
+    file_data: &S,
     object_header: &ObjectHeader,
     name: &str,
     offset_size: u8,
     length_size: u8,
 ) -> Result<Option<LinkTarget>, FormatError> {
-    if is_v1_group(object_header) {
     let Some(sym_msg) = object_header
         .messages
         .iter()
@@ -175,43 +258,254 @@ fn find_symbolic_link(
         return Ok(None);
     };
     let stm = SymbolTableMessage::parse(&sym_msg.data, offset_size)?;
-        return group_v1::find_v1_soft_link(file_data, &stm, name, offset_size, length_size)
+    group_v1::find_v1_soft_link_in(file_data, &stm, name, offset_size, length_size)
-            .map(|target| target.map(|target_path| LinkTarget::Soft { target_path }));
+        .map(|target| target.map(|target_path| LinkTarget::Soft { target_path }))
-    }
+}
-    if !is_v2_group(object_header) {
+
-        return Ok(None);
+/// B-tree v2 record type of a dense group's link name index.
-    }
+const LINK_NAME_INDEX: u8 = 5;
-    let is_symbolic = |t: &LinkTarget| !matches!(t, LinkTarget::Hard { .. });
+
+/// The links called `name` in a v2 group (a valid group has at most one),
+/// in storage order: header message order for a compact group, name index
+/// order for a dense one.
+///
+/// In dense storage the link name index (a v2 B-tree of lookup3 name
+/// hashes, record type 5) is descended to the records with the name's hash,
+/// and only their links are read from the heap — O(log n) instead of every
+/// link. libhdf5 orders records with equal hashes by name; all of them are
+/// read and compared here, so that order does not matter. An index of
+/// another type is scanned in full.
+fn links_named<S: Storage + ?Sized>(
+    file_data: &S,
+    object_header: &ObjectHeader,
+    name: &str,
+    offset_size: u8,
+    length_size: u8,
+) -> Result<Vec<LinkMessage>, FormatError> {
+    let mut found = Vec::new();
     let link_info = find_link_info(object_header, offset_size)?;
-    let mut found = None;
+    let Some(fh_addr) = link_info.fractal_heap_address else {
-    if let Some(fh_addr) = link_info.fractal_heap_address {
+        for msg in &object_header.messages {
+            if msg.msg_type == MessageType::Link
+                && let Some(link) = parse_link(&msg.data, offset_size)?
+                && link.name == name
+            {
+                found.push(link);
+            }
+        }
+        return Ok(found);
+    };
+
+    let fh = dense_heap(file_data, &link_info, fh_addr, offset_size, length_size)?;
+    let btree_addr = link_info
+        .btree_name_index_address
+        .ok_or_else(|| FormatError::PathNotFound(String::from("no B-tree v2 name index")))?;
+    let btree_hdr = BTreeV2Header::parse_in(
+        file_data,
+        checked_addr(btree_addr)?,
+        offset_size,
+        length_size,
+    )?;
+    if btree_hdr.tree_type != LINK_NAME_INDEX {
         for_each_dense_link(
             file_data,
             &link_info,
             fh_addr,
             offset_size,
             length_size,
+            false,
             |link| {
-                if link.name == name && is_symbolic(&link.link_target) {
+                if link.name == name {
-                    found = Some(link.link_target);
+                    found.push(link);
                 }
             },
         )?;
-    } else {
+        return Ok(found);
-        for msg in &object_header.messages {
+    }
-            if msg.msg_type == MessageType::Link {
+
-                let Some(link) = parse_link(&msg.data, offset_size)? else {
+    // Record: hash(4) + heap ID.
+    let hash = jenkins_lookup3(name.as_bytes());
+    let records = find_btree_v2_records_in(file_data, &btree_hdr, offset_size, &mut |r| {
+        match r.get(..4) {
+            Some(h) => u32::from_le_bytes([h[0], h[1], h[2], h[3]]).cmp(&hash),
+            // Too short to hold a hash (a corrupt record size): never a match.
+            None => core::cmp::Ordering::Less,
+        }
+    })?;
+    let id_len = usize::from(fh.heap_id_length);
+    for record in &records {
+        let Some(id_bytes) = record.data.get(4..4 + id_len) else {
             continue;
         };
-                if link.name == name && is_symbolic(&link.link_target) {
+        let link_data = fh.read_managed_object_in(file_data, id_bytes, offset_size)?;
-                    found = Some(link.link_target);
+        if let Some(link) = parse_link(&link_data, offset_size)?
-                }
+            && link.name == name
-            }
+        {
+            found.push(link);
         }
     }
     Ok(found)
 }
 
+/// The link called `name` in a v2 group, if any.
+///
+/// A valid group has at most one; libhdf5 cannot create two. If a damaged
+/// or hand-made group has several, the first wins and the rest are
+/// ignored, whatever their kind and even if the first cannot be followed.
+/// That is libhdf5's rule for a compact group (`H5G__compact_lookup` stops
+/// at the first Link message of that name; h5py then fails to open a
+/// dangling first link although a later one resolves). For a dense group
+/// "first" is first in name index order; libhdf5 binary-searches the index
+/// and may land on another of several exact duplicates. The listing
+/// ([`resolve_group_children`]), [`resolve_child`] and path resolution all
+/// apply this rule, so they agree.
+fn first_link_named<S: Storage + ?Sized>(
+    file_data: &S,
+    object_header: &ObjectHeader,
+    name: &str,
+    offset_size: u8,
+    length_size: u8,
+) -> Result<Option<LinkMessage>, FormatError> {
+    Ok(
+        links_named(file_data, object_header, name, offset_size, length_size)?
+            .into_iter()
+            .next(),
+    )
+}
+
+/// The link [`resolve_path_any`] follows for one path component `name` of
+/// the group with header `object_header`: a hard link (as `Hard`), else a
+/// soft or external link of that name, else `None`. Fails with
+/// `PathNotFound` if the object is not a group.
+fn lookup_link<S: Storage + ?Sized>(
+    file_data: &S,
+    object_header: &ObjectHeader,
+    name: &str,
+    offset_size: u8,
+    length_size: u8,
+) -> Result<Option<LinkTarget>, FormatError> {
+    if is_v1_group(object_header) {
+        // Down the group's B-tree, as libhdf5 looks a name up; only when
+        // that does not find a hard link of that name is every entry read
+        // (a soft link, a group whose B-tree is out of order). A storage
+        // error (a read a restartable storage has not fetched yet) is
+        // returned as is: reading every entry would not get further.
+        if let Some(sym_msg) = object_header
+            .messages
+            .iter()
+            .find(|m| m.msg_type == MessageType::SymbolTable)
+        {
+            let stm = SymbolTableMessage::parse(&sym_msg.data, offset_size)?;
+            match group_v1::find_v1_entry(file_data, &stm, name, offset_size, length_size) {
+                Ok(Some(e)) if e.object_header_address != u64::MAX => {
+                    return Ok(Some(LinkTarget::Hard {
+                        object_header_address: e.object_header_address,
+                    }));
+                }
+                Err(e @ FormatError::Storage(_)) => return Err(e),
+                _ => {}
+            }
+        }
+        let entries = resolve_group_entries(file_data, object_header, offset_size, length_size)?;
+        if let Some(e) = entries
+            .iter()
+            .find(|e| e.name == name && e.object_header_address != u64::MAX)
+        {
+            return Ok(Some(LinkTarget::Hard {
+                object_header_address: e.object_header_address,
+            }));
+        }
+        return find_v1_symbolic_link(file_data, object_header, name, offset_size, length_size);
+    }
+    if !is_v2_group(object_header) {
+        return Err(FormatError::PathNotFound(String::from(
+            "object header is not a group",
+        )));
+    }
+    Ok(
+        first_link_named(file_data, object_header, name, offset_size, length_size)?
+            .map(|link| link.link_target)
+            .filter(|t| {
+                !matches!(
+                    t,
+                    LinkTarget::Hard {
+                        object_header_address: u64::MAX
+                    }
+                )
+            }),
+    )
+}
+
+/// The object header address of the child called `name` of the group at
+/// `group_address`: the address [`resolve_group_children`] lists under that
+/// name, or `PathNotFound` if it lists none.
+///
+/// A dense group's child is found through its link name index (see
+/// [`links_named`]) and only the named link is read and, if it is a soft
+/// link, followed — not every link in the group. A v1 group is listed.
+pub fn resolve_child(
+    file_data: &[u8],
+    superblock: &Superblock,
+    group_address: u64,
+    name: &str,
+) -> Result<u64, FormatError> {
+    resolve_child_core(file_data, superblock, group_address, name)
+}
+
+/// [`resolve_child`] over any [`Storage`]. One with the whole file in memory
+/// is read as the slice, by code compiled in this crate (see
+/// [`crate::storage`], "Slice entry points").
+#[inline]
+pub fn resolve_child_in<S: Storage + ?Sized>(
+    file_data: &S,
+    superblock: &Superblock,
+    group_address: u64,
+    name: &str,
+) -> Result<u64, FormatError> {
+    match file_data.as_contiguous() {
+        Some(all) => resolve_child(all, superblock, group_address, name),
+        None => resolve_child_core(file_data, superblock, group_address, name),
+    }
+}
+
+fn resolve_child_core<S: Storage + ?Sized>(
+    file_data: &S,
+    superblock: &Superblock,
+    group_address: u64,
+    name: &str,
+) -> Result<u64, FormatError> {
+    let os = superblock.offset_size;
+    let ls = superblock.length_size;
+    let not_found = || FormatError::PathNotFound(String::from(name));
+    let header = ObjectHeader::parse_in(file_data, checked_addr(group_address)?, os, ls)?;
+    if !is_v2_group(&header) || is_v1_group(&header) {
+        return group_children(file_data, superblock, group_address, false)?
+            .into_iter()
+            .find(|e| e.name == name)
+            .map(|e| e.object_header_address)
+            .ok_or_else(not_found);
+    }
+    // The first link of that name only, as the listing (see
+    // `first_link_named`).
+    match first_link_named(file_data, &header, name, os, ls)?.map(|l| l.link_target) {
+        Some(LinkTarget::Hard {
+            object_header_address,
+        }) => Ok(object_header_address),
+        Some(LinkTarget::Soft { target_path }) => {
+            match resolve_path_from_in(file_data, superblock, group_address, &target_path) {
+                // Left out of the listing: dangling, cyclic, or in another file.
+                Err(
+                    FormatError::PathNotFound(_)
+                    | FormatError::NestingDepthExceeded
+                    | FormatError::ExternalLinkUnsupported { .. },
+                ) => Err(not_found()),
+                other => other,
+            }
+        }
+        Some(LinkTarget::External { .. }) | None => Err(not_found()),
+    }
+}
+
 /// Find and parse the Link Info message from an object header.
 fn find_link_info(
     object_header: &ObjectHeader,
@@ -254,6 +548,29 @@ pub fn resolve_path_any(
     file_data: &[u8],
     superblock: &Superblock,
     path: &str,
+) -> Result<u64, FormatError> {
+    resolve_path_any_core(file_data, superblock, path)
+}
+
+/// [`resolve_path_any`] over any [`Storage`]. One with the whole file in memory
+/// is read as the slice, by code compiled in this crate (see
+/// [`crate::storage`], "Slice entry points").
+#[inline]
+pub fn resolve_path_any_in<S: Storage + ?Sized>(
+    file_data: &S,
+    superblock: &Superblock,
+    path: &str,
+) -> Result<u64, FormatError> {
+    match file_data.as_contiguous() {
+        Some(all) => resolve_path_any(all, superblock, path),
+        None => resolve_path_any_core(file_data, superblock, path),
+    }
+}
+
+fn resolve_path_any_core<S: Storage + ?Sized>(
+    file_data: &S,
+    superblock: &Superblock,
+    path: &str,
 ) -> Result<u64, FormatError> {
     resolve_path_following_links(
         file_data,
@@ -272,6 +589,16 @@ pub fn resolve_path_from(
     superblock: &Superblock,
     group_address: u64,
     path: &str,
+) -> Result<u64, FormatError> {
+    resolve_path_from_in(file_data, superblock, group_address, path)
+}
+
+/// [`resolve_path_from`] over any [`Storage`].
+pub fn resolve_path_from_in<S: Storage + ?Sized>(
+    file_data: &S,
+    superblock: &Superblock,
+    group_address: u64,
+    path: &str,
 ) -> Result<u64, FormatError> {
     let start = if path.starts_with('/') {
         superblock.root_group_address
@@ -295,10 +622,45 @@ pub fn resolve_group_children(
     file_data: &[u8],
     superblock: &Superblock,
     group_address: u64,
+) -> Result<Vec<GroupEntry>, FormatError> {
+    resolve_group_children_core(file_data, superblock, group_address)
+}
+
+/// [`resolve_group_children`] over any [`Storage`]. One with the whole file in memory
+/// is read as the slice, by code compiled in this crate (see
+/// [`crate::storage`], "Slice entry points").
+#[inline]
+pub fn resolve_group_children_in<S: Storage + ?Sized>(
+    file_data: &S,
+    superblock: &Superblock,
+    group_address: u64,
+) -> Result<Vec<GroupEntry>, FormatError> {
+    match file_data.as_contiguous() {
+        Some(all) => resolve_group_children(all, superblock, group_address),
+        None => resolve_group_children_core(file_data, superblock, group_address),
+    }
+}
+
+fn resolve_group_children_core<S: Storage + ?Sized>(
+    file_data: &S,
+    superblock: &Superblock,
+    group_address: u64,
+) -> Result<Vec<GroupEntry>, FormatError> {
+    group_children(file_data, superblock, group_address, true)
+}
+
+/// [`resolve_group_children`]; with `hint_headers`, every child's object
+/// header is hinted (see [`Storage::hint`]) as soon as its address is
+/// known, for a listing whose children are opened next.
+fn group_children<S: Storage + ?Sized>(
+    file_data: &S,
+    superblock: &Superblock,
+    group_address: u64,
+    hint_headers: bool,
 ) -> Result<Vec<GroupEntry>, FormatError> {
     let os = superblock.offset_size;
     let ls = superblock.length_size;
-    let header = ObjectHeader::parse(file_data, group_address as usize, os, ls)?;
+    let header = ObjectHeader::parse_in(file_data, checked_addr(group_address)?, os, ls)?;
 
     let mut entries = Vec::new();
     let mut soft = Vec::new();
@@ -309,13 +671,22 @@ pub fn resolve_group_children(
             .find(|m| m.msg_type == MessageType::SymbolTable)
             .ok_or_else(|| FormatError::PathNotFound(String::from("no symbol table message")))?;
         let stm = SymbolTableMessage::parse(&sym_msg.data, os)?;
-        let all = group_v1::resolve_v1_group_entries(file_data, &stm, os, ls)?;
+        let all = group_v1::v1_group_entries(file_data, &stm, os, ls, hint_headers)?;
+        if all.iter().any(|e| e.name.is_empty()) {
+            return Err(FormatError::InvalidLinkName);
+        }
         if all.iter().any(group_v1::is_v1_soft_link) {
-            soft = group_v1::v1_soft_links(file_data, &stm, os, ls)?;
+            soft = group_v1::v1_soft_links_in(file_data, &stm, os, ls)?;
         }
         entries.extend(all.into_iter().filter(|e| !group_v1::is_v1_soft_link(e)));
     } else if is_v2_group(&header) {
-        let mut visit = |link: LinkMessage| match link.link_target {
+        // Only the first link of each name counts (see `first_link_named`).
+        let mut seen = BTreeSet::new();
+        let mut visit = |link: LinkMessage| {
+            if !seen.insert(link.name.clone()) {
+                return;
+            }
+            match link.link_target {
                 LinkTarget::Hard {
                     object_header_address,
                 } => entries.push(GroupEntry {
@@ -325,10 +696,11 @@ pub fn resolve_group_children(
                 }),
                 LinkTarget::Soft { target_path } => soft.push((link.name, target_path)),
                 LinkTarget::External { .. } => {}
+            }
         };
         let link_info = find_link_info(&header, os)?;
         if let Some(fh_addr) = link_info.fractal_heap_address {
-            for_each_dense_link(file_data, &link_info, fh_addr, os, ls, visit)?;
+            for_each_dense_link(file_data, &link_info, fh_addr, os, ls, hint_headers, visit)?;
         } else {
             for msg in &header.messages {
                 if msg.msg_type == MessageType::Link
@@ -341,7 +713,7 @@ pub fn resolve_group_children(
     }
 
     for (name, target) in soft {
-        match resolve_path_from(file_data, superblock, group_address, &target) {
+        match resolve_path_from_in(file_data, superblock, group_address, &target) {
             Ok(object_header_address) => entries.push(GroupEntry {
                 name,
                 object_header_address,
@@ -364,8 +736,8 @@ pub fn resolve_group_children(
 const MAX_SOFT_LINK_DEPTH: u8 = 16;
 
 /// Walk `path` from the group at `start`, following soft links.
-fn resolve_path_following_links(
+fn resolve_path_following_links<S: Storage + ?Sized>(
-    file_data: &[u8],
+    file_data: &S,
     superblock: &Superblock,
     start: u64,
     path: &str,
@@ -383,24 +755,22 @@ fn resolve_path_following_links(
     let ls = superblock.length_size;
 
     let mut current_addr = start;
-    let mut current_header = ObjectHeader::parse(file_data, start as usize, os, ls)?;
+    let mut current_header = ObjectHeader::parse_in(file_data, checked_addr(start)?, os, ls)?;
 
     for (i, component) in components.iter().enumerate() {
-        let entries = resolve_group_entries(file_data, &current_header, os, ls)?;
+        match lookup_link(file_data, &current_header, component, os, ls)? {
-
+            Some(LinkTarget::Hard {
-        let found = entries
+                object_header_address,
-            .iter()
+            }) => {
-            .find(|e| e.name == *component && e.object_header_address != u64::MAX);
-        match found {
-            Some(entry) => {
                 if i == components.len() - 1 {
-                    return Ok(entry.object_header_address);
+                    return Ok(object_header_address);
                 }
-                current_addr = entry.object_header_address;
+                current_addr = object_header_address;
-                current_header = ObjectHeader::parse(file_data, current_addr as usize, os, ls)?;
+                current_header =
+                    ObjectHeader::parse_in(file_data, checked_addr(current_addr)?, os, ls)?;
             }
-            None => {
+            found => {
-                return match find_symbolic_link(file_data, &current_header, component, os, ls)? {
+                return match found {
                     Some(LinkTarget::Soft { target_path }) => {
                         if depth >= MAX_SOFT_LINK_DEPTH {
                             return Err(FormatError::NestingDepthExceeded);
@@ -436,8 +806,8 @@ fn resolve_path_following_links(
 }
 
 /// Resolve group entries from an object header, auto-detecting v1 vs v2.
-fn resolve_group_entries(
+fn resolve_group_entries<S: Storage + ?Sized>(
-    file_data: &[u8],
+    file_data: &S,
     object_header: &ObjectHeader,
     offset_size: u8,
     length_size: u8,
@@ -450,9 +820,11 @@ fn resolve_group_entries(
             .find(|m| m.msg_type == MessageType::SymbolTable)
             .ok_or_else(|| FormatError::PathNotFound(String::from("no symbol table message")))?;
         let stm = SymbolTableMessage::parse(&sym_msg.data, offset_size)?;
-        group_v1::resolve_v1_group_entries(file_data, &stm, offset_size, length_size)
+        // A lookup: an entry with an empty name (which fails a listing) is
+        // skipped by the name comparison, as in libhdf5.
+        group_v1::v1_group_entries(file_data, &stm, offset_size, length_size, false)
     } else if is_v2_group(object_header) {
-        resolve_v2_group_entries(file_data, object_header, offset_size, length_size)
+        resolve_v2_group_entries_in(file_data, object_header, offset_size, length_size)
     } else {
         Err(FormatError::PathNotFound(String::from(
             "object header is not a group",
@@ -633,6 +1005,86 @@ mod tests {
         assert_eq!(values, vec![22.5, 23.1, 21.8]);
     }
 
+    /// `v1_groups_400.h5` from its superblock on (it has a user block).
+    fn v1_groups_400() -> (Vec<u8>, Superblock) {
+        let all: &[u8] = include_bytes!("../tests/fixtures/v1_groups_400.h5");
+        let data = all[signature::find_signature(all).unwrap()..].to_vec();
+        let sb = Superblock::parse(&data, 0).unwrap();
+        (data, sb)
+    }
+
+    /// Every child of a v1 group resolves by name, down the group's B-tree,
+    /// to the address the listing gives, reading a small part of what the
+    /// listing reads; a name the group does not hold is not found.
+    #[test]
+    fn v1_lookup_down_the_btree_agrees_with_the_listing() {
+        let (data, sb) = v1_groups_400();
+        let children = resolve_group_children(&data, &sb, sb.root_group_address).unwrap();
+        assert_eq!(children.len(), 401);
+        for c in &children {
+            let path = format!("/{}", c.name);
+            assert_eq!(
+                resolve_path_any(&data, &sb, &path).unwrap(),
+                c.object_header_address,
+                "{path}"
+            );
+        }
+        for missing in ["/g0400", "/a", "/g", "/g00000", "/zz", "/x0"] {
+            assert!(
+                matches!(
+                    resolve_path_any(&data, &sb, missing),
+                    Err(FormatError::PathNotFound(_))
+                ),
+                "{missing}"
+            );
+        }
+        let st = crate::storage::CountingStorage::new(data.clone());
+        resolve_group_children_in(&st, &sb, sb.root_group_address).unwrap();
+        let listing = st.bytes_read();
+        st.reset();
+        let last = children.last().unwrap();
+        assert_eq!(
+            resolve_path_any_in(&st, &sb, &format!("/{}", last.name)).unwrap(),
+            last.object_header_address
+        );
+        assert!(
+            st.bytes_read() * 8 < listing,
+            "lookup read {} bytes, listing {listing}",
+            st.bytes_read()
+        );
+    }
+
+    /// A v1 group whose B-tree is out of name order (a name changed in the
+    /// heap so that it sorts past every key) is still looked up by reading
+    /// every entry, as before the lookup went down the B-tree.
+    #[test]
+    fn v1_lookup_falls_back_when_the_btree_is_out_of_order() {
+        let (mut data, sb) = v1_groups_400();
+        let at: Vec<usize> = data
+            .windows(6)
+            .enumerate()
+            .filter(|(_, w)| *w == b"g0200\0")
+            .map(|(i, _)| i)
+            .collect();
+        assert_eq!(at.len(), 1, "one heap string");
+        data[at[0]] = b'~';
+        let children = resolve_group_children(&data, &sb, sb.root_group_address).unwrap();
+        let moved = children.iter().find(|c| c.name == "~0200").unwrap();
+        assert_eq!(
+            resolve_path_any(&data, &sb, "/~0200").unwrap(),
+            moved.object_header_address
+        );
+        assert!(resolve_path_any(&data, &sb, "/g0200").is_err());
+        for c in &children {
+            let path = format!("/{}", c.name);
+            assert_eq!(
+                resolve_path_any(&data, &sb, &path).unwrap(),
+                c.object_header_address,
+                "{path}"
+            );
+        }
+    }
+
     #[test]
     fn path_not_found_v2() {
         let file_data: &[u8] = include_bytes!("../tests/fixtures/v2_groups.h5");

crates/clawhdf5-format/src/lane_partition.rs

@@ -112,7 +112,8 @@ pub fn partition(
 
     for idx in 0..num_items {
         let h = fxhash_combine(seed, idx as u64);
-        let lane = (h % num_lanes as u64) as usize;
+        // Below `num_lanes`, so it fits.
+        let lane = crate::addr::saturating_usize(h % num_lanes as u64);
         lanes[lane].push(idx);
     }
 

crates/clawhdf5-format/src/lib.rs

@@ -57,10 +57,13 @@
 #[cfg(not(feature = "std"))]
 extern crate alloc;
 
+pub mod addr;
 pub mod attribute;
 pub mod attribute_info;
 pub mod btree_v1;
+mod btree_v1_write;
 pub mod btree_v2;
+mod btree_v2_write;
 mod bulk_alloc;
 pub mod checksum;
 pub mod chunk_cache;
@@ -86,11 +89,15 @@ pub mod filters;
 mod filters_bitshuffle;
 #[cfg(feature = "blosc")]
 pub mod filters_blosc;
+#[cfg(feature = "blosc2")]
+pub mod filters_blosc2;
 #[cfg(feature = "bzip2")]
 mod filters_bzip2;
 #[cfg(feature = "lzf")]
 pub mod filters_lzf;
 mod filters_szip;
+#[cfg(feature = "zfp")]
+pub mod filters_zfp;
 pub mod fixed_array;
 pub mod float16;
 pub mod fractal_heap;
@@ -101,9 +108,11 @@ pub mod group_v1;
 pub mod group_v2;
 #[cfg(feature = "parallel")]
 pub mod lane_partition;
+pub mod libver;
 pub mod link_info;
 pub mod link_message;
 pub mod local_heap;
+pub mod lookup_stats;
 pub mod message_type;
 pub mod metadata_cache;
 pub mod metadata_index;
@@ -117,7 +126,9 @@ pub mod property_list;
 pub mod selection;
 pub mod shared_message;
 pub mod signature;
+pub mod storage;
 pub mod superblock;
+pub mod superblock_ext;
 pub mod symbol_table;
 #[cfg(all(
     test,

crates/clawhdf5-format/src/libver.rs

@@ -0,0 +1,93 @@
+//! Library version bounds for writing: which HDF5 releases can read a file.
+//!
+//! libhdf5 picks the version of every object it writes from the file's
+//! *low* bound (`H5Pset_libver_bounds`; h5py's `libver=`): the oldest
+//! format version that holds the object, but never older than the one the
+//! low bound names. The *high* bound caps it: a feature that needs a newer
+//! format than the high bound is an error. [`LibVer`] names the same
+//! releases, and [`crate::file_writer::FileWriter::libver_bounds`] sets them.
+//!
+//! What the low bound changes in what clawhdf5 writes:
+//!
+//! | | low [`LibVer::V18`] | low [`LibVer::V110`] or later (the default) |
+//! |---|---|---|
+//! | superblock | version 2 | version 3 |
+//! | data layout message | version 3 | version 4 |
+//! | chunk index | version-1 B-tree (every chunked dataset) | single chunk, Fixed Array, Extensible Array or version-2 B-tree, as libhdf5 picks |
+//!
+//! Everything else (version-2 object headers, link and group-info messages,
+//! dense storage in fractal heaps with version-2 B-trees, filter pipeline
+//! version 2, fill value version 3, datatype versions up to 3) is the same
+//! and already readable by HDF5 1.8.
+//!
+//! What the high bound refuses: anything that needs 1.10 (virtual datasets,
+//! the paged file-space strategy) above [`LibVer::V18`], the 1.12 reference
+//! types (datatype version 4) above [`LibVer::V110`], and HDF5 2.0's native
+//! complex numbers (datatype version 5) above [`LibVer::V114`].
+//! `libver_bounds(LibVer::V18, LibVer::V18)` therefore writes a file HDF5
+//! 1.8 can read, or fails.
+
+use core::fmt;
+
+/// An HDF5 library release, as a bound on the file format versions a writer
+/// may use (libhdf5's `H5F_libver_t`). Ordered oldest first.
+///
+/// There is no `Earliest`: clawhdf5 cannot write the pre-1.8 format
+/// (symbol-table groups, version-1 object headers).
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
+#[non_exhaustive]
+pub enum LibVer {
+    /// HDF5 1.8 (`H5F_LIBVER_V18`, h5py `'v108'`).
+    V18,
+    /// HDF5 1.10 (`H5F_LIBVER_V110`, h5py `'v110'`).
+    V110,
+    /// HDF5 1.12 (`H5F_LIBVER_V112`, h5py `'v112'`).
+    V112,
+    /// HDF5 1.14 (`H5F_LIBVER_V114`, h5py `'v114'`).
+    V114,
+    /// HDF5 2.0 (`H5F_LIBVER_V200`).
+    V200,
+    /// The newest format this build of clawhdf5 writes
+    /// (`H5F_LIBVER_LATEST`, h5py `'latest'`).
+    Latest,
+}
+
+impl LibVer {
+    /// The release a datatype message of this version first appeared in:
+    /// versions 1-3 are readable by HDF5 1.8, 4 needs 1.12, 5 needs 2.0.
+    pub(crate) fn for_datatype_version(version: u8) -> Self {
+        match version {
+            0..=3 => LibVer::V18,
+            4 => LibVer::V112,
+            _ => LibVer::V200,
+        }
+    }
+}
+
+impl fmt::Display for LibVer {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.write_str(match self {
+            LibVer::V18 => "1.8",
+            LibVer::V110 => "1.10",
+            LibVer::V112 => "1.12",
+            LibVer::V114 => "1.14",
+            LibVer::V200 => "2.0",
+            LibVer::Latest => "latest",
+        })
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn ordered_oldest_first() {
+        assert!(LibVer::V18 < LibVer::V110);
+        assert!(LibVer::V114 < LibVer::V200);
+        assert!(LibVer::V200 < LibVer::Latest);
+        assert_eq!(LibVer::for_datatype_version(3), LibVer::V18);
+        assert_eq!(LibVer::for_datatype_version(4), LibVer::V112);
+        assert_eq!(LibVer::for_datatype_version(5), LibVer::V200);
+    }
+}

crates/clawhdf5-format/src/link_message.rs

@@ -3,6 +3,7 @@
 #[cfg(not(feature = "std"))]
 use alloc::{string::String, vec::Vec};
 
+use crate::addr::to_usize;
 use crate::datatype::CharacterSet;
 use crate::error::FormatError;
 
@@ -247,7 +248,7 @@ impl LinkMessage {
         };
 
         // Link name length
-        let name_len = read_offset(data, pos, name_size_field_width)? as usize;
+        let name_len = to_usize(read_offset(data, pos, name_size_field_width)?)?;
         pos += name_size_field_width as usize;
 
         // Link name

crates/clawhdf5-format/src/local_heap.rs

@@ -3,7 +3,9 @@
 #[cfg(not(feature = "std"))]
 use alloc::string::String;
 
+use crate::addr::to_usize;
 use crate::error::FormatError;
+use crate::storage::{Storage, len_usize, read_exact_at};
 
 /// Parsed HDF5 Local Heap header.
 #[derive(Debug, Clone)]
@@ -16,21 +18,6 @@ pub struct LocalHeap {
     pub data_segment_address: u64,
 }
 
-/// Checks that `[offset, offset + needed)` fits within `data`, guarding the
-/// addition against `usize` overflow from a crafted near-`usize::MAX` offset.
-fn ensure_len(data: &[u8], offset: usize, needed: usize) -> Result<(), FormatError> {
-    if offset
-        .checked_add(needed)
-        .is_none_or(|end| end > data.len())
-    {
-        return Err(FormatError::UnexpectedEof {
-            expected: offset.saturating_add(needed),
-            available: data.len(),
-        });
-    }
-    Ok(())
-}
-
 fn read_offset(data: &[u8], pos: usize, size: u8) -> Result<u64, FormatError> {
     let s = size as usize;
     if pos.checked_add(s).is_none_or(|end| end > data.len()) {
@@ -50,6 +37,10 @@ fn read_offset(data: &[u8], pos: usize, size: u8) -> Result<u64, FormatError> {
     })
 }
 
+/// First read of a name on a backend without the file in memory: most link
+/// names are shorter than this.
+const NAME_READ_START: usize = 64;
+
 impl LocalHeap {
     /// Parse a local heap header at the given offset in the file data.
     pub fn parse(
@@ -57,12 +48,24 @@ impl LocalHeap {
         offset: usize,
         offset_size: u8,
         length_size: u8,
+    ) -> Result<LocalHeap, FormatError> {
+        Self::parse_in(file_data, offset as u64, offset_size, length_size)
+    }
+
+    /// [`Self::parse`] over any [`Storage`]: one read of the header.
+    pub fn parse_in<S: Storage + ?Sized>(
+        file: &S,
+        offset: u64,
+        offset_size: u8,
+        length_size: u8,
     ) -> Result<LocalHeap, FormatError> {
         // signature(4) + version(1) + reserved(3) = 8, then length_size*2 + offset_size
         let ls = length_size as usize;
         let os = offset_size as usize;
         let total = 8 + ls * 2 + os;
-        ensure_len(file_data, offset, total)?;
+        let header = read_exact_at(file, offset, total)?;
+        let file_data: &[u8] = &header;
+        let offset = 0usize;
 
         if &file_data[offset..offset + 4] != b"HEAP" {
             return Err(FormatError::InvalidLocalHeapSignature);
@@ -99,6 +102,16 @@ impl LocalHeap {
     /// The end of the list is `H5HL_FREE_NULL` (1); an all-ones value (the
     /// undefined address) is accepted as "no free list" too.
     pub fn validate_free_list(&self, file_data: &[u8], length_size: u8) -> Result<(), FormatError> {
+        self.validate_free_list_in(file_data, length_size)
+    }
+
+    /// [`Self::validate_free_list`] over any [`Storage`]: two small reads
+    /// per free block.
+    pub fn validate_free_list_in<S: Storage + ?Sized>(
+        &self,
+        file: &S,
+        length_size: u8,
+    ) -> Result<(), FormatError> {
         const FREE_NULL: u64 = 1;
         let ls = length_size as usize;
         let undefined = if ls >= 8 {
@@ -123,11 +136,12 @@ impl LocalHeap {
                 .and_then(|a| usize::try_from(a).ok())
                 .ok_or(FormatError::InvalidLocalHeapFreeList)?;
             let block_offset = next;
-            next = read_offset(file_data, at, length_size)?;
+            next = read_offset(&read_exact_at(file, at as u64, ls)?, 0, length_size)?;
             if next == 0 {
                 return Err(FormatError::InvalidLocalHeapFreeList);
             }
-            let block_size = read_offset(file_data, at + ls, length_size)?;
+            let block_size =
+                read_offset(&read_exact_at(file, (at + ls) as u64, ls)?, 0, length_size)?;
             if block_offset
                 .checked_add(block_size)
                 .is_none_or(|end| end > size)
@@ -140,43 +154,65 @@ impl LocalHeap {
 
     /// Read a null-terminated string from the heap's data segment at the given byte offset.
     pub fn read_string(&self, file_data: &[u8], string_offset: u64) -> Result<String, FormatError> {
-        let seg_addr = self.data_segment_address as usize;
+        self.read_string_in(file_data, string_offset)
+    }
+
+    /// [`Self::read_string`] over any [`Storage`]: one read of up to 64
+    /// bytes for a short name, more (each four times the last) up to the end
+    /// of the data segment for a longer one.
+    pub fn read_string_in<S: Storage + ?Sized>(
+        &self,
+        file: &S,
+        string_offset: u64,
+    ) -> Result<String, FormatError> {
+        let file_len = len_usize(file);
+        let seg_addr = to_usize(self.data_segment_address)?;
         let str_start =
             seg_addr
-                .checked_add(string_offset as usize)
+                .checked_add(to_usize(string_offset)?)
                 .ok_or(FormatError::Overflow(
                     "local heap seg_addr + string_offset overflow".into(),
                 ))?;
         let seg_end = seg_addr
-            .checked_add(self.data_segment_size as usize)
+            .checked_add(to_usize(self.data_segment_size)?)
             .ok_or(FormatError::Overflow(
                 "local heap seg_addr + data_segment_size overflow".into(),
             ))?;
 
-        if str_start >= file_data.len() || str_start >= seg_end {
+        if str_start >= file_len || str_start >= seg_end {
             return Err(FormatError::UnexpectedEof {
                 expected: str_start + 1,
-                available: file_data.len(),
+                available: file_len,
             });
         }
 
-        // Find null terminator
+        // Find the null terminator, which lies before the end of the data
-        let search_end = seg_end.min(file_data.len());
+        // segment (or of the file). In memory that is one borrowed slice;
-        let mut end = str_start;
+        // otherwise the bytes are read in growing pieces, so a name costs a
-        while end < search_end && file_data[end] != 0 {
+        // read of about its own length, not of the rest of the segment
-            end += 1;
+        // (whose size is an untrusted header field).
+        let search_end = seg_end.min(file_len);
+        let total = search_end - str_start;
+        let mut want = if file.as_contiguous().is_some() {
+            total
+        } else {
+            total.min(NAME_READ_START)
+        };
+        loop {
+            let rest = read_exact_at(file, str_start as u64, want)?;
+            if let Some(len) = rest.iter().position(|&b| b == 0) {
+                let s = core::str::from_utf8(&rest[..len])
+                    .map_err(|_| FormatError::InvalidLocalHeapSignature)?;
+                return Ok(String::from(s));
             }
-
+            if want == total {
-        if end >= search_end {
                 return Err(FormatError::UnexpectedEof {
-                expected: end + 1,
+                    expected: search_end + 1,
                     available: search_end,
                 });
             }
-
+            want = want.saturating_mul(4).min(total);
-        let s = core::str::from_utf8(&file_data[str_start..end])
+        }
-            .map_err(|_| FormatError::InvalidLocalHeapSignature)?;
-        Ok(String::from(s))
     }
 }
 
@@ -345,4 +381,77 @@ mod tests {
         let err = LocalHeap::parse(&file, 0, 8, 8).unwrap_err();
         assert_eq!(err, FormatError::InvalidLocalHeapVersion(1));
     }
+
+    /// Header, free list and strings read identically through a
+    /// `read_at`-only storage, for every truncation of the file.
+    #[test]
+    fn storage_reads_match_slice_reads() {
+        use crate::storage::CountingStorage;
+        let plain = build_heap_file(0, 64, &["", "alpha", "beta"], 8, 8);
+        // A free block of 16 bytes at segment offset 12, ending the list.
+        let mut free = build_heap_file(0, 64, &["", "alpha", "beta", &"x".repeat(20)], 8, 8);
+        free[16..24].copy_from_slice(&12u64.to_le_bytes());
+        free[64 + 12..64 + 20].copy_from_slice(&1u64.to_le_bytes());
+        free[64 + 20..64 + 28].copy_from_slice(&16u64.to_le_bytes());
+        let mut bad_free = free.clone();
+        bad_free[64 + 20..64 + 28].copy_from_slice(&99u64.to_le_bytes());
+        for full in [plain, free, bad_free] {
+            for cut in 0..=full.len() {
+                let f = &full[..cut];
+                let storage = CountingStorage::new(f.to_vec());
+                let want = LocalHeap::parse(f, 0, 8, 8);
+                let got = LocalHeap::parse_in(&storage, 0, 8, 8);
+                assert_eq!(format!("{got:?}"), format!("{want:?}"));
+                let Ok(heap) = want else { continue };
+                assert_eq!(
+                    heap.validate_free_list_in(&storage, 8),
+                    heap.validate_free_list(f, 8)
+                );
+                for off in [0u64, 1, 2, 6, 7, 11, 100] {
+                    assert_eq!(heap.read_string_in(&storage, off), heap.read_string(f, off));
+                }
+            }
+        }
+    }
+
+    /// Names of every length around the first read's size, and one with no
+    /// terminator, read identically through a `read_at`-only storage; a
+    /// short name in a heap whose header claims a huge data segment costs
+    /// one small read, not a read of the rest of the file.
+    #[test]
+    fn long_names_and_hostile_segment_sizes() {
+        use crate::storage::CountingStorage;
+        let names: Vec<String> = [0usize, 1, 63, 64, 65, 255, 256, 257, 1000, 5000]
+            .iter()
+            .map(|&n| "n".repeat(n))
+            .collect();
+        let refs: Vec<&str> = names.iter().map(String::as_str).collect();
+        let mut file = build_heap_file(0, 64, &refs, 8, 8);
+        let heap = LocalHeap::parse(&file, 0, 8, 8).unwrap();
+        let storage = CountingStorage::new(file.clone());
+        let mut off = 0u64;
+        for name in &names {
+            let got = heap.read_string_in(&storage, off);
+            assert_eq!(got, heap.read_string(&file, off));
+            assert_eq!(got.unwrap(), *name);
+            off += name.len() as u64 + 1;
+        }
+        // The last name loses its terminator: both report the same error.
+        let seg_end = 64 + heap.data_segment_size as usize;
+        file[seg_end - 1] = b'n';
+        let storage = CountingStorage::new(file.clone());
+        let last = off - names[names.len() - 1].len() as u64 - 1;
+        let want = heap.read_string(&file, last);
+        assert!(want.is_err());
+        assert_eq!(heap.read_string_in(&storage, last), want);
+
+        // A 64 MiB file whose heap claims a data segment reaching its end.
+        let mut big = build_heap_file(0, 64, &["short", "names"], 8, 8);
+        big.resize(64 << 20, 0);
+        big[8..16].copy_from_slice(&((64u64 << 20) - 64).to_le_bytes());
+        let heap = LocalHeap::parse(&big, 0, 8, 8).unwrap();
+        let storage = CountingStorage::new(big.clone());
+        assert_eq!(heap.read_string_in(&storage, 6).unwrap(), "names");
+        assert_eq!((storage.reads(), storage.bytes_read()), (1, 64));
+    }
 }

crates/clawhdf5-format/src/lookup_stats.rs

@@ -0,0 +1,31 @@
+//! Work counters for tests of lookup cost (feature `lookup-stats`).
+//!
+//! Counts fractal-heap objects read — each is one link or attribute message
+//! decoded out of a dense group or dense attribute storage — so a test can
+//! check that finding one name reads a handful of them, not the whole group.
+//! Per thread, so tests running in parallel do not see each other's reads.
+//! Without the feature the counting compiles to nothing.
+
+#[cfg(feature = "lookup-stats")]
+std::thread_local! {
+    static HEAP_OBJECTS: core::cell::Cell<u64> = const { core::cell::Cell::new(0) };
+}
+
+/// Record one heap object read.
+#[inline(always)]
+pub(crate) fn heap_object_read() {
+    #[cfg(feature = "lookup-stats")]
+    HEAP_OBJECTS.with(|c| c.set(c.get() + 1));
+}
+
+/// Heap objects read on this thread since the last [`reset`].
+#[cfg(feature = "lookup-stats")]
+pub fn heap_objects_read() -> u64 {
+    HEAP_OBJECTS.with(core::cell::Cell::get)
+}
+
+/// Zero this thread's counters.
+#[cfg(feature = "lookup-stats")]
+pub fn reset() {
+    HEAP_OBJECTS.with(|c| c.set(0));
+}

crates/clawhdf5-format/src/object_header.rs

@@ -1,12 +1,16 @@
 //! HDF5 Object Header parsing (v1 and v2).
 
 #[cfg(not(feature = "std"))]
-use alloc::vec::Vec;
+use alloc::{boxed::Box, collections::BTreeSet, vec::Vec};
+#[cfg(feature = "std")]
+use std::collections::BTreeSet;
 
 use byteorder::{ByteOrder, LittleEndian};
 
+use crate::addr::to_usize;
 use crate::error::FormatError;
 use crate::message_type::MessageType;
+use crate::storage::{Storage, Window, len_usize, read_exact_at};
 
 /// OHDR signature for v2 object headers.
 const OHDR_SIGNATURE: [u8; 4] = *b"OHDR";
@@ -75,42 +79,128 @@ fn read_offset(data: &[u8], pos: usize, size: u8) -> Result<u64, FormatError> {
     })
 }
 
+/// The kind of object an object header describes.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum ObjectClass {
+    /// A group: the header has a Symbol Table or a Link Info message.
+    Group,
+    /// A dataset: the header has a Datatype and a Dataspace message.
+    Dataset,
+    /// A committed (named) datatype: a Datatype message, no Dataspace.
+    NamedDatatype,
+}
+
 impl ObjectHeader {
+    /// The kind of object this header describes, decided as libhdf5 decides
+    /// it (`H5O__obj_class_real`): group first (a Symbol Table or Link Info
+    /// message), then dataset (a Datatype *and* a Dataspace message — not a
+    /// Data Layout message), then named datatype (a Datatype message).
+    /// `None` when none applies; libhdf5 then cannot open the object
+    /// ("unable to determine object type").
+    ///
+    /// A header with a Datatype and a Data Layout message but no Dataspace
+    /// is a named datatype to libhdf5, not a dataset.
+    pub fn object_class(&self) -> Option<ObjectClass> {
+        let has = |t: MessageType| self.messages.iter().any(|m| m.msg_type == t);
+        if has(MessageType::SymbolTable) || has(MessageType::LinkInfo) {
+            Some(ObjectClass::Group)
+        } else if has(MessageType::Datatype) && has(MessageType::Dataspace) {
+            Some(ObjectClass::Dataset)
+        } else if has(MessageType::Datatype) {
+            Some(ObjectClass::NamedDatatype)
+        } else {
+            None
+        }
+    }
+
     /// Parse an object header at the given offset in the data buffer.
     ///
     /// `offset_size` and `length_size` come from the superblock.
+    #[inline]
     pub fn parse(
         data: &[u8],
         offset: usize,
         offset_size: u8,
         length_size: u8,
     ) -> Result<ObjectHeader, FormatError> {
-        ensure_len(data, offset, 4)?;
+        Self::parse_slice(data, offset as u64, offset_size, length_size)
-        if data[offset..offset + 4] == OHDR_SIGNATURE {
+    }
-            Self::parse_v2(data, offset, offset_size, length_size)
+
-        } else {
+    /// [`Self::parse`] over any [`Storage`].
-            Self::parse_v1(data, offset, offset_size, length_size)
+    ///
+    /// Reads the prefix (at most [`V2_PREFIX_MAX`] bytes, signature
+    /// included), then each chunk as one bounded read, continuation chunks
+    /// included. A storage with the whole file in memory is parsed as its
+    /// slice, by code compiled in this crate (see
+    /// [`crate::storage`], "Slice entry points").
+    #[inline]
+    pub fn parse_in<S: Storage + ?Sized>(
+        file: &S,
+        offset: u64,
+        offset_size: u8,
+        length_size: u8,
+    ) -> Result<ObjectHeader, FormatError> {
+        match file.as_contiguous() {
+            Some(all) => Self::parse_slice(all, offset, offset_size, length_size),
+            None => Self::parse_storage(file, offset, offset_size, length_size),
         }
     }
 
-    fn parse_v1(
+    /// [`Self::parse_storage`] for the slice, compiled in this crate: the
+    /// one copy [`Self::parse`] and [`Self::parse_in`] (in memory) call.
+    fn parse_slice(
         data: &[u8],
-        offset: usize,
+        offset: u64,
+        offset_size: u8,
+        length_size: u8,
+    ) -> Result<ObjectHeader, FormatError> {
+        Self::parse_storage(data, offset, offset_size, length_size)
+    }
+
+    fn parse_storage<S: Storage + ?Sized>(
+        file: &S,
+        offset: u64,
+        offset_size: u8,
+        length_size: u8,
+    ) -> Result<ObjectHeader, FormatError> {
+        // The first chunk is read once the prefix says how long it is: say
+        // so (see `Storage::hint`), for a storage that fetches between
+        // attempts.
+        if file.as_contiguous().is_none() {
+            file.hint(offset, OBJECT_HEADER_HINT_LEN);
+        }
+        // The longest prefix of either version, in one read. It holds the
+        // whole prefix or ends at the end of the file, so its bounds checks
+        // are the whole-file ones.
+        let prefix = Window::read(file, offset, V2_PREFIX_MAX)?;
+        prefix.ensure(0, 4)?;
+        if prefix.bytes[..4] == OHDR_SIGNATURE {
+            Self::parse_v2(file, offset, &prefix, offset_size, length_size)
+        } else {
+            Self::parse_v1(file, offset, &prefix, offset_size, length_size)
+        }
+    }
+
+    fn parse_v1<S: Storage + ?Sized>(
+        file: &S,
+        offset: u64,
+        prefix: &Window<'_>,
         offset_size: u8,
         length_size: u8,
     ) -> Result<ObjectHeader, FormatError> {
         // version(1) + reserved(1) + num_messages(2) + ref_count(4) + header_size(4) = 12
         // then pad to 8-byte alignment from start of header
-        ensure_len(data, offset, 12)?;
+        prefix.ensure(0, 12)?;
+        let prefix = &prefix.bytes[..12];
 
-        let version = data[offset];
+        let version = prefix[0];
         if version != 1 {
             return Err(FormatError::InvalidObjectHeaderVersion(version));
         }
 
-        let num_messages = LittleEndian::read_u16(&data[offset + 2..offset + 4]) as usize;
+        let num_messages = LittleEndian::read_u16(&prefix[2..4]) as usize;
-        let reference_count = LittleEndian::read_u32(&data[offset + 4..offset + 8]);
+        let reference_count = LittleEndian::read_u32(&prefix[4..8]);
-        let header_data_size = LittleEndian::read_u32(&data[offset + 8..offset + 12]) as usize;
+        let header_data_size = LittleEndian::read_u32(&prefix[8..12]) as usize;
 
         // libhdf5 (H5O__prefix_deserialize): a header with messages needs room
         // for at least one message header, and one without has an empty chunk.
@@ -128,19 +218,19 @@ impl ObjectHeader {
             .checked_add(12 + padding)
             .ok_or(FormatError::UnexpectedEof {
                 expected: usize::MAX,
-                available: data.len(),
+                available: len_usize(file),
             })?;
 
-        ensure_len(data, msg_start, header_data_size)?;
+        // parse_v1_chunk reads the chunk, with the bounds check that was here.
-
+        // The prefix's count (NIL messages included, capped: it is untrusted)
-        let mut messages = Vec::new();
+        // sizes the list once instead of growing it message by message.
+        let mut messages = Vec::with_capacity(num_messages.min(64));
         let chunk0_count = Self::parse_v1_chunk(
-            data,
+            file,
             msg_start,
             header_data_size,
             offset_size,
             length_size,
-            MAX_V1_CONTINUATION_DEPTH,
             &mut messages,
         )?;
         // libhdf5 reads every message in the first chunk and refuses a header
@@ -174,24 +264,70 @@ impl ObjectHeader {
     /// 8; libhdf5 refuses a message that is not aligned, that runs past the
     /// end of the chunk, or leftover bytes too few for a message header (a
     /// "gap", which only version 2 allows).
-    #[allow(clippy::too_many_arguments)]
+    ///
-    fn parse_v1_chunk(
+    /// Continuation chunks are read in the order their messages are found,
-        data: &[u8],
+    /// as `H5O_protect` loads them (so the messages keep libhdf5's order):
-        offset: usize,
+    /// a queue of (address, length) pairs, each chunk read, parsed and
+    /// released before the next, so only one chunk buffer is alive at a
+    /// time whatever the storage. Every chunk must start at a new address
+    /// (else a cycle), and the chunks together may be no larger than the
+    /// file, so the bytes read stay within the file's size; a header of
+    /// more than [`MAX_V1_CHUNKS`] chunks is refused.
+    fn parse_v1_chunk<S: Storage + ?Sized>(
+        file: &S,
+        offset: u64,
         length: usize,
         offset_size: u8,
         length_size: u8,
-        depth_remaining: u16,
         messages: &mut Vec<HeaderMessage>,
     ) -> Result<usize, FormatError> {
-        if depth_remaining == 0 {
+        // The chunks found so far are also the queue of chunks to read.
-            return Err(FormatError::NestingDepthExceeded);
+        let mut spans = ChunkSpans::new(file.len(), offset, length)?;
+        let mut chunk0_count = 0usize;
+        let mut next = 0usize;
+        let hints = file.as_contiguous().is_none();
+        while let Some((chunk_offset, chunk_length)) = spans.get(next) {
+            let chunk = read_exact_at(file, chunk_offset, chunk_length)?;
+            let known = spans.len;
+            let count =
+                Self::parse_v1_messages(&chunk, offset_size, length_size, messages, &mut spans)?;
+            // The continuation chunks this one names are read next.
+            if hints {
+                for i in known..spans.len {
+                    if let Some((o, l)) = spans.get(i) {
+                        file.hint(o, l);
+                    }
+                }
+            }
+            // Only the first chunk's messages are held to the prefix count.
+            if next == 0 {
+                chunk0_count = count;
+            }
+            next += 1;
+        }
+        Ok(chunk0_count)
     }
-        ensure_len(data, offset, length)?;
-        let end = offset + length;
-        let mut pos = offset;
-        let mut count = 0usize;
 
+    /// The messages of one version-1 chunk: each checked and appended to
+    /// `messages` (NIL ones dropped), each continuation added to `spans`.
+    /// Returns how many messages (NIL ones included) the chunk holds.
+    ///
+    /// Inlined into the chunk loop: kept out of line (`#[inline(never)]`,
+    /// 4313917), the call cost `ObjectHeader::parse` about 2.5 ns per
+    /// header, 4% on small version-1 headers (A/B builds, 2026-09-27; see
+    /// `BENCHMARKS.md`). Without an attribute the compiler keeps it out of
+    /// line too.
+    #[inline]
+    fn parse_v1_messages(
+        data: &[u8],
+        offset_size: u8,
+        length_size: u8,
+        messages: &mut Vec<HeaderMessage>,
+        spans: &mut ChunkSpans,
+    ) -> Result<usize, FormatError> {
+        let end = data.len();
+        let mut pos = 0usize;
+        let mut count = 0usize;
         while pos < end {
             if end - pos < V1_MSG_HEADER_SIZE {
                 return Err(FormatError::InvalidObjectHeader(
@@ -215,7 +351,6 @@ impl ObjectHeader {
             let body = &data[pos..pos + msg_data_size];
             check_message(1, msg_type_raw, msg_flags, body, offset_size, length_size)?;
             count += 1;
-
             let msg_type = MessageType::from_u16(msg_type_raw);
             if msg_type != MessageType::Nil {
                 messages.push(HeaderMessage {
@@ -226,34 +361,34 @@ impl ObjectHeader {
                     data: body.to_vec(),
                 });
             }
-            pos += msg_data_size;
+            // Queue continuations (v1 continuation chunks are just raw
-
-            // Follow continuations (v1 continuation chunks are just raw
             // messages, no signature); check_message has checked the body.
             if msg_type == MessageType::ObjectHeaderContinuation {
-                let cont_offset = read_offset(body, 0, offset_size)? as usize;
+                let cont_offset = read_offset(body, 0, offset_size)?;
-                let cont_length = read_offset(body, offset_size as usize, length_size)? as usize;
+                let cont_length = to_usize(read_offset(body, offset_size as usize, length_size)?)?;
-                Self::parse_v1_chunk(
+                spans.add(cont_offset, cont_length)?;
-                    data,
-                    cont_offset,
-                    cont_length,
-                    offset_size,
-                    length_size,
-                    depth_remaining - 1,
-                    messages,
-                )?;
             }
+            pos += msg_data_size;
         }
-
         Ok(count)
     }
 
-    fn parse_v2(
+    fn parse_v2<S: Storage + ?Sized>(
-        data: &[u8],
+        file: &S,
-        offset: usize,
+        offset: u64,
+        prefix: &Window<'_>,
         offset_size: u8,
         length_size: u8,
     ) -> Result<ObjectHeader, FormatError> {
+        // `ensure_len` checks positions relative to the header against the
+        // prefix window and reports them as the whole-file check did, with
+        // absolute positions and the file's length.
+        let data: &[u8] = &prefix.bytes;
+        let file_len = len_usize(file);
+        let base = usize::try_from(offset).unwrap_or(usize::MAX);
+        let abs = |rel: usize| base.saturating_add(rel);
+        let ensure_len = |_: &[u8], rel: usize, needed: usize| prefix.ensure(rel, needed);
+        let offset = 0usize;
         // signature(4) + version(1) + flags(1) = 6
         ensure_len(data, offset, 6)?;
 
@@ -306,7 +441,7 @@ impl ObjectHeader {
             _ => unreachable!(),
         };
         ensure_len(data, pos, chunk_size_width as usize)?;
-        let chunk0_size = read_offset(data, pos, chunk_size_width)? as usize;
+        let chunk0_size = to_usize(read_offset(data, pos, chunk_size_width)?)?;
         pos += chunk_size_width as usize;
         // Bit 2: attribute creation order tracked → messages include creation order field
         let has_creation_order = flags & 0x04 != 0;
@@ -318,15 +453,20 @@ impl ObjectHeader {
         }
 
         let chunk0_msg_start = pos;
-        let chunk0_msg_end = pos
+        let Some(chunk0_abs_end) = abs(pos).checked_add(chunk0_size) else {
-            .checked_add(chunk0_size)
+            return Err(FormatError::UnexpectedEof {
-            .ok_or(FormatError::UnexpectedEof {
                 expected: usize::MAX,
-                available: data.len(),
+                available: file_len,
-            })?;
+            });
+        };
+        let chunk0_msg_end = chunk0_abs_end - base;
+
+        // The whole first chunk, prefix to checksum, in one read (its
+        // bounds check is the one on the checksum's 4 bytes).
+        let chunk0 = read_exact_at(file, base as u64, chunk0_msg_end.saturating_add(4))?;
+        let data: &[u8] = &chunk0;
 
         // Validate checksum: from OHDR signature through all messages (before checksum)
-        ensure_len(data, chunk0_msg_end, 4)?;
         #[cfg(feature = "checksum")]
         {
             let stored = LittleEndian::read_u32(&data[chunk0_msg_end..chunk0_msg_end + 4]);
@@ -353,16 +493,26 @@ impl ObjectHeader {
             &mut continuations,
         )?;
 
-        // Follow continuations (limit to prevent cycles in malformed data)
+        // Follow continuations, one chunk buffer at a time. A chunk address
-        let mut cont_remaining = 256u16;
+        // seen twice is a cycle in malformed data, and the chunks may add up
-        while let Some((cont_offset, cont_length)) = continuations.pop() {
+        // to no more than the file; a valid header can have many chunks (libhdf5 adds one
-            if cont_remaining == 0 {
+        // whenever a message no longer fits), up to the same bound as a
-                return Err(FormatError::NestingDepthExceeded);
+        // version-1 header.
+        let mut spans = ChunkSpans::new(file.len(), base as u64, chunk0_msg_end.saturating_add(4))?;
+        // The continuation chunks a chunk names are read next (see
+        // `Storage::hint`).
+        let hints = file.as_contiguous().is_none();
+        if hints {
+            for &(o, l) in &continuations {
+                file.hint(o as u64, l);
             }
-            cont_remaining -= 1;
+        }
+        while let Some((cont_offset, cont_length)) = continuations.pop() {
+            spans.add(cont_offset as u64, cont_length)?;
+            let known = continuations.len();
             Self::parse_v2_continuation(
-                data,
+                file,
-                cont_offset,
+                cont_offset as u64,
                 cont_length,
                 has_creation_order,
                 offset_size,
@@ -370,6 +520,11 @@ impl ObjectHeader {
                 &mut messages,
                 &mut continuations,
             )?;
+            if hints {
+                for &(o, l) in &continuations[known..] {
+                    file.hint(o as u64, l);
+                }
+            }
         }
 
         Ok(ObjectHeader {
@@ -439,8 +594,8 @@ impl ObjectHeader {
             let msg_type = MessageType::from_u16(msg_type_raw);
             if msg_type == MessageType::ObjectHeaderContinuation {
                 // check_message has checked the body holds both fields.
-                let cont_off = read_offset(body, 0, offset_size)? as usize;
+                let cont_off = to_usize(read_offset(body, 0, offset_size)?)?;
-                let cont_len = read_offset(body, offset_size as usize, length_size)? as usize;
+                let cont_len = to_usize(read_offset(body, offset_size as usize, length_size)?)?;
                 continuations.push((cont_off, cont_len));
             } else if msg_type == MessageType::Nil {
                 null_count += 1;
@@ -461,9 +616,9 @@ impl ObjectHeader {
     }
 
     #[allow(clippy::too_many_arguments)]
-    fn parse_v2_continuation(
+    fn parse_v2_continuation<S: Storage + ?Sized>(
-        data: &[u8],
+        file: &S,
-        offset: usize,
+        offset: u64,
         length: usize,
         has_creation_order: bool,
         offset_size: u8,
@@ -472,7 +627,9 @@ impl ObjectHeader {
         continuations: &mut Vec<(usize, usize)>,
     ) -> Result<(), FormatError> {
         // OCHK signature(4) + messages + checksum(4)
-        ensure_len(data, offset, length)?;
+        let chunk = read_exact_at(file, offset, length)?;
+        let data: &[u8] = &chunk;
+        let offset = 0usize;
         if length < 8 {
             return Err(FormatError::UnexpectedEof {
                 expected: 8,
@@ -513,11 +670,119 @@ impl ObjectHeader {
     }
 }
 
+/// What an object header is hinted to take before its prefix is read (see
+/// [`Storage::hint`]): the first chunk of a typical dataset's header. A
+/// longer header is read all the same.
+pub(crate) const OBJECT_HEADER_HINT_LEN: usize = 512;
+
+/// Longest version-2 object header prefix: signature(4) + version(1) +
+/// flags(1) + times(16) + attribute phase change(4) + chunk-0 size(8).
+const V2_PREFIX_MAX: usize = 34;
+
 /// Size of a version-1 message header: type(2) + size(2) + flags(1) + reserved(3).
 const V1_MSG_HEADER_SIZE: usize = 8;
 
-/// How deep version-1 continuation chunks may chain (malformed-data guard).
+/// The chunks of one object header read so far, in the order they were
-const MAX_V1_CONTINUATION_DEPTH: u16 = 32;
+/// found (which is the order version-1 chunks are read in). A chunk starting
+/// where another did is a cycle. Chunks of a valid header do not overlap, so
+/// together they are no larger than the file; a header whose chunks add up
+/// to more is refused, which bounds what its chunks can make a reader read
+/// (a crafted chain of chunks each nested in the last would otherwise read
+/// the file over and over). Overlap itself is not refused: libhdf5 reads
+/// such headers (`cve-2025-7067.h5` has one).
+///
+/// Almost every header has at most a few chunks, and this runs once per
+/// header, so the first [`INLINE_CHUNKS`] live in an inline array and are
+/// checked for cycles by a scan; only a longer header allocates (the rest
+/// of the list, and a set of starts). Allocating a queue and a set for
+/// every header made parsing 401 small headers 1.8x slower.
+struct ChunkSpans {
+    inline: [(u64, usize); INLINE_CHUNKS],
+    /// Chunks after the first [`INLINE_CHUNKS`], and every chunk start.
+    spill: Option<Box<SpilledSpans>>,
+    /// How many chunks there are.
+    len: usize,
+    /// Bytes of the chunks so far, and the most they may add up to.
+    total: u64,
+    budget: u64,
+}
+
+/// The chunks of a [`ChunkSpans`] beyond its inline ones.
+struct SpilledSpans {
+    chunks: Vec<(u64, usize)>,
+    starts: BTreeSet<u64>,
+}
+
+/// How many chunks [`ChunkSpans`] holds without allocating.
+const INLINE_CHUNKS: usize = 8;
+
+impl ChunkSpans {
+    #[inline]
+    fn new(file_len: u64, start: u64, len: usize) -> Result<Self, FormatError> {
+        let mut s = Self {
+            inline: [(0, 0); INLINE_CHUNKS],
+            spill: None,
+            len: 0,
+            total: 0,
+            budget: file_len,
+        };
+        s.add(start, len)?;
+        Ok(s)
+    }
+
+    /// Record the chunk `len` bytes at `start`.
+    #[inline]
+    fn add(&mut self, start: u64, len: usize) -> Result<(), FormatError> {
+        self.total = self.total.saturating_add(len as u64);
+        if self.len < INLINE_CHUNKS {
+            if self.inline[..self.len].iter().any(|&(s, _)| s == start) {
+                return Err(FormatError::NestingDepthExceeded);
+            }
+            self.inline[self.len] = (start, len);
+        } else {
+            self.add_spilled(start, len)?;
+        }
+        self.len += 1;
+        if self.total > self.budget {
+            return Err(FormatError::InvalidObjectHeader(
+                "object header chunks larger than the file",
+            ));
+        }
+        Ok(())
+    }
+
+    #[cold]
+    #[inline(never)]
+    fn add_spilled(&mut self, start: u64, len: usize) -> Result<(), FormatError> {
+        let inline = &self.inline;
+        let spill = self.spill.get_or_insert_with(|| {
+            Box::new(SpilledSpans {
+                chunks: Vec::new(),
+                starts: inline.iter().map(|&(s, _)| s).collect(),
+            })
+        });
+        if !spill.starts.insert(start) || self.len >= MAX_V1_CHUNKS {
+            return Err(FormatError::NestingDepthExceeded);
+        }
+        spill.chunks.push((start, len));
+        Ok(())
+    }
+
+    /// The `i`th chunk recorded.
+    #[inline]
+    fn get(&self, i: usize) -> Option<(u64, usize)> {
+        if i < INLINE_CHUNKS {
+            (i < self.len).then(|| self.inline[i])
+        } else {
+            self.spill.as_ref()?.chunks.get(i - INLINE_CHUNKS).copied()
+        }
+    }
+}
+
+/// Most chunks a version-1 object header may have (malformed-data guard;
+/// libhdf5 has no limit, and a header that gains one continuation chunk per
+/// attribute added can have many).
+const MAX_V1_CHUNKS: usize = 1 << 16;
 
 /// Every defined version-2 object header status flag (libhdf5
 /// `H5O_HDR_ALL_FLAGS`): chunk-0 size width (bits 0-1), attribute creation
@@ -662,6 +927,54 @@ fn check_message(
 mod tests {
     use super::*;
 
+    fn header_with(types: &[MessageType]) -> ObjectHeader {
+        ObjectHeader {
+            version: 2,
+            messages: types
+                .iter()
+                .map(|&msg_type| HeaderMessage {
+                    msg_type,
+                    size: 0,
+                    flags: 0,
+                    creation_order: None,
+                    data: Vec::new(),
+                })
+                .collect(),
+            reference_count: None,
+            flags: 0,
+            access_time: None,
+            modification_time: None,
+            change_time: None,
+            birth_time: None,
+        }
+    }
+
+    #[test]
+    fn object_class_follows_libhdf5() {
+        use MessageType::*;
+        let class = |t: &[MessageType]| header_with(t).object_class();
+        assert_eq!(
+            class(&[Datatype, Dataspace, DataLayout]),
+            Some(ObjectClass::Dataset)
+        );
+        // A Data Layout message does not make a dataset without a dataspace
+        // (cve-2024-33874 `/Dset1`: h5py opens it as a named datatype).
+        assert_eq!(
+            class(&[Datatype, DataLayout]),
+            Some(ObjectClass::NamedDatatype)
+        );
+        assert_eq!(class(&[Datatype]), Some(ObjectClass::NamedDatatype));
+        // Group messages win over dataset messages.
+        assert_eq!(
+            class(&[Datatype, Dataspace, SymbolTable]),
+            Some(ObjectClass::Group)
+        );
+        assert_eq!(class(&[LinkInfo]), Some(ObjectClass::Group));
+        // Link messages alone are not a group; nothing is not an object.
+        assert_eq!(class(&[Link]), None);
+        assert_eq!(class(&[]), None);
+    }
+
     // Helper: build a v1 object header with given messages
     fn build_v1_header(
         messages: &[(u16, &[u8], u8)], // (type, data, flags)
@@ -673,13 +986,13 @@ mod tests {
         let mut msg_bytes = Vec::new();
         for (mtype, mdata, mflags) in messages {
             // v1 message sizes are multiples of 8 (the data is zero-padded).
-            let padded = mdata.len().div_ceil(8) * 8;
+            let padded = <[u8]>::len(mdata).div_ceil(8) * 8;
             msg_bytes.extend_from_slice(&mtype.to_le_bytes()); // type(2)
             msg_bytes.extend_from_slice(&(padded as u16).to_le_bytes()); // size(2)
             msg_bytes.push(*mflags); // flags(1)
             msg_bytes.extend_from_slice(&[0u8; 3]); // reserved(3)
             msg_bytes.extend_from_slice(mdata); // data
-            msg_bytes.resize(msg_bytes.len() + padded - mdata.len(), 0);
+            msg_bytes.resize(msg_bytes.len() + padded - <[u8]>::len(mdata), 0);
         }
 
         let mut buf = Vec::new();
@@ -775,6 +1088,176 @@ mod tests {
         assert_eq!(hdr.messages[1].data[..2], [5, 6]);
     }
 
+    /// A version-1 header whose continuation chunks form a chain: chunk k
+    /// holds a Dataspace message `[k]` and the continuation to chunk k + 1.
+    /// With `cycle`, the last chunk points back at the first continuation
+    /// chunk.
+    fn v1_chain(n: usize, cycle: bool) -> Vec<u8> {
+        // Each continuation chunk: dataspace (8 + 8) + continuation (8 + 16).
+        let chunk_len = 40u64;
+        let first = 64u64;
+        let cont = |addr: u64| {
+            let mut b = addr.to_le_bytes().to_vec();
+            b.extend_from_slice(&chunk_len.to_le_bytes());
+            b
+        };
+        let mut data = build_v1_header(&[(0x0010, &cont(first)[..], 0)], 8, 8);
+        data.resize(first as usize, 0);
+        for k in 0..n {
+            let mut c = Vec::new();
+            c.extend_from_slice(&1u16.to_le_bytes());
+            c.extend_from_slice(&8u16.to_le_bytes());
+            c.extend_from_slice(&[0; 4]);
+            c.extend_from_slice(&(k as u64).to_le_bytes());
+            let next = if k + 1 < n {
+                first + (k as u64 + 1) * chunk_len
+            } else if cycle {
+                first
+            } else {
+                // The last chunk ends in a NIL message instead.
+                c.extend_from_slice(&[0, 0, 16, 0, 0, 0, 0, 0]);
+                c.extend_from_slice(&[0; 16]);
+                data.extend_from_slice(&c);
+                continue;
+            };
+            c.extend_from_slice(&0x10u16.to_le_bytes());
+            c.extend_from_slice(&16u16.to_le_bytes());
+            c.extend_from_slice(&[0; 4]);
+            c.extend_from_slice(&cont(next));
+            data.extend_from_slice(&c);
+        }
+        data
+    }
+
+    /// libhdf5 reads any chain of continuation chunks (a header grows one
+    /// per attribute added when full); the reader used to stop at 32.
+    #[test]
+    fn long_v1_continuation_chains_are_read() {
+        let data = v1_chain(200, false);
+        let hdr = ObjectHeader::parse(&data, 0, 8, 8).unwrap();
+        let spaces: Vec<u8> = hdr
+            .messages
+            .iter()
+            .filter(|m| m.msg_type == MessageType::Dataspace)
+            .map(|m| m.data[0])
+            .collect();
+        assert_eq!(spaces, (0..200).map(|k| k as u8).collect::<Vec<_>>());
+    }
+
+    /// A crafted version-1 header whose continuation chunks nest: each
+    /// chunk's continuation message points at the rest of that chunk. Read
+    /// depth-first with every enclosing chunk kept alive, from storage that
+    /// hands out owned buffers, it read n^2 bytes and held them all at once
+    /// (a 192 KB file read 768 MB). Chunks adding up to more than the file
+    /// are refused, and the bytes read stay within the file's size.
+    #[test]
+    fn nested_v1_continuation_chunks_are_bounded() {
+        use crate::storage::CountingStorage;
+        let n = 2000u64;
+        let a = 64u64;
+        let cont = |addr: u64, len: u64| {
+            let mut m = vec![0x10, 0, 16, 0, 0, 0, 0, 0];
+            m.extend_from_slice(&addr.to_le_bytes());
+            m.extend_from_slice(&len.to_le_bytes());
+            m
+        };
+        // Prefix: version 1, one message, reference count 1, 24 bytes.
+        let mut buf = vec![1, 0, 1, 0, 1, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0];
+        buf.extend_from_slice(&cont(a, 24 * n));
+        buf.resize(a as usize, 0);
+        for k in 0..n {
+            if k + 1 < n {
+                buf.extend_from_slice(&cont(a + 24 * (k + 1), 24 * (n - k - 1)));
+            } else {
+                buf.extend_from_slice(&[0, 0, 16, 0, 0, 0, 0, 0]);
+                buf.extend_from_slice(&[0; 16]);
+            }
+        }
+        let len = buf.len() as u64;
+        let s = CountingStorage::new(buf);
+        assert!(matches!(
+            ObjectHeader::parse_in(&s, 0, 8, 8),
+            Err(FormatError::InvalidObjectHeader(
+                "object header chunks larger than the file"
+            ))
+        ));
+        assert!(
+            s.bytes_read() <= 2 * len,
+            "read {} of {len}",
+            s.bytes_read()
+        );
+    }
+
+    /// libhdf5 reads a continuation chunk that overlaps the chunk holding
+    /// its message (`cve-2025-7067.h5` has one), and so does this reader.
+    #[test]
+    fn overlapping_v1_continuation_chunk_is_read() {
+        // Chunk 0 (at 16): continuation (24 bytes), then a NIL message at
+        // 40; the continuation chunk is that NIL message's 8-byte header.
+        let mut cont = 40u64.to_le_bytes().to_vec();
+        cont.extend_from_slice(&8u64.to_le_bytes());
+        let data = build_v1_header(&[(0x0010, &cont[..], 0), (0x0000, &[][..], 0)], 8, 8);
+        let hdr = ObjectHeader::parse(&data, 0, 8, 8).unwrap();
+        assert_eq!(hdr.messages.len(), 1);
+    }
+
+    /// A valid chain over owned-buffer storage reads each chunk once.
+    #[test]
+    fn long_v1_chain_reads_each_chunk_once() {
+        use crate::storage::CountingStorage;
+        let data = v1_chain(3000, false);
+        let len = data.len() as u64;
+        let s = CountingStorage::new(data);
+        let hdr = ObjectHeader::parse_in(&s, 0, 8, 8).unwrap();
+        assert_eq!(
+            hdr.messages
+                .iter()
+                .filter(|m| m.msg_type == MessageType::Dataspace)
+                .count(),
+            3000
+        );
+        assert!(s.bytes_read() <= len, "read {} of {len}", s.bytes_read());
+    }
+
+    /// Continuation chunks are read in the order their messages are found
+    /// (libhdf5's `H5O_protect`), so a chunk's messages follow every
+    /// message of the chunk before, not the continuation message.
+    #[test]
+    fn v1_continuation_messages_keep_libhdf5_order() {
+        // Chunk 0: continuation to A, dataspace [1]; A: dataspace [2].
+        let a = 64u64;
+        let mut cont = a.to_le_bytes().to_vec();
+        cont.extend_from_slice(&16u64.to_le_bytes());
+        let mut data = build_v1_header(&[(0x0010, &cont[..], 0), (0x0001, &[1; 8][..], 0)], 8, 8);
+        data.resize(a as usize, 0);
+        data.extend_from_slice(&[1, 0, 8, 0, 0, 0, 0, 0]);
+        data.extend_from_slice(&[2; 8]);
+        let hdr = ObjectHeader::parse(&data, 0, 8, 8).unwrap();
+        let spaces: Vec<u8> = hdr
+            .messages
+            .iter()
+            .filter(|m| m.msg_type == MessageType::Dataspace)
+            .map(|m| m.data[0])
+            .collect();
+        assert_eq!(spaces, [1, 2]);
+    }
+
+    #[test]
+    fn v1_continuation_cycles_are_refused() {
+        // Within the inline chunk list, and past it (the cycle returns to
+        // an inline chunk once the list has spilled).
+        for n in [5, 7, 8, 9, 40] {
+            let data = v1_chain(n, true);
+            assert!(
+                matches!(
+                    ObjectHeader::parse(&data, 0, 8, 8),
+                    Err(FormatError::NestingDepthExceeded)
+                ),
+                "{n} chunks"
+            );
+        }
+    }
+
     #[test]
     fn parse_v1_unknown_message_ok() {
         let messages = [(0x00FFu16, &[0xAA, 0xBB][..], 0u8)];
@@ -1199,4 +1682,56 @@ mod tests {
         let err = ObjectHeader::parse(&data, 0, 8, 8).unwrap_err();
         assert!(matches!(err, FormatError::UnexpectedEof { .. }));
     }
+
+    /// Every header, and every truncation of it, parses to the same result
+    /// (or the same error) through a `read_at`-only storage as from a slice;
+    /// a header in one chunk takes two reads (prefix, chunk).
+    #[test]
+    fn parse_in_matches_slice_parse() {
+        use crate::storage::CountingStorage;
+        let mut headers = vec![
+            build_v1_header(&[], 8, 8),
+            build_v1_header(&[(0x0001, &[1, 2, 3], 0), (0x0003, &[9; 8], 0)], 8, 8),
+            build_v2_header(0x00, &[(0x01, &[42], 0)], None),
+            build_v2_header(0x03, &[(0x01, &[1, 2], 0), (0x03, &[3], 0)], None),
+            build_v2_header(0x24, &[(0x01, &[1], 0)], Some((1, 2, 3, 4))),
+            build_v2_header(0x35, &[(0x01, &[1], 0)], Some((5, 6, 7, 8))),
+        ];
+        // A v2 header with a continuation chunk at 256.
+        let mut ochk = OCHK_SIGNATURE.to_vec();
+        ochk.extend_from_slice(&[0x03, 2, 0, 0, 0xDE, 0xAD]);
+        let sum = crate::checksum::jenkins_lookup3(&ochk);
+        ochk.extend_from_slice(&sum.to_le_bytes());
+        let mut cont = 256u64.to_le_bytes().to_vec();
+        cont.extend_from_slice(&(ochk.len() as u64).to_le_bytes());
+        let main = build_v2_header(0x00, &[(0x01, &[42], 0), (0x10, &cont, 0)], None);
+        let mut with_cont = vec![0u8; 256 + ochk.len()];
+        with_cont[..main.len()].copy_from_slice(&main);
+        with_cont[256..].copy_from_slice(&ochk);
+        headers.push(with_cont);
+
+        for h in headers {
+            for at in [0usize, 3] {
+                for cut in 0..=h.len() {
+                    let mut f = vec![0u8; at];
+                    f.extend_from_slice(&h[..cut]);
+                    if at == 0 && cut == h.len() {
+                        f.resize(f.len() + 64, 0);
+                    }
+                    let want = ObjectHeader::parse(&f, at, 8, 8);
+                    let storage = CountingStorage::new(f.clone());
+                    let got = ObjectHeader::parse_in(&storage, at as u64, 8, 8);
+                    assert_eq!(
+                        format!("{got:?}"),
+                        format!("{want:?}"),
+                        "at {at}, cut {cut}"
+                    );
+                }
+            }
+        }
+        let one_chunk = build_v2_header(0x00, &[(0x01, &[42], 0)], None);
+        let storage = CountingStorage::new(one_chunk);
+        ObjectHeader::parse_in(&storage, 0, 8, 8).unwrap();
+        assert_eq!(storage.reads(), 2);
+    }
 }

crates/clawhdf5-format/src/object_header_writer.rs

@@ -12,9 +12,16 @@ use crate::message_type::MessageType;
 /// its size truncated to 16 bits produced files libhdf5 refuses.
 pub const MAX_MESSAGE_SIZE: usize = u16::MAX as usize;
 
+/// Object header flags: attribute creation order tracked (each message
+/// then carries a 2-byte creation order) and indexed.
+const OHDR_ATTR_CRT_ORDER_TRACKED: u8 = 0x04;
+const OHDR_ATTR_CRT_ORDER_INDEXED: u8 = 0x08;
+
 /// Writer for v2 object headers with proper checksums.
 pub struct ObjectHeaderWriter {
-    messages: Vec<(MessageType, Vec<u8>, u8)>, // (type, data, msg_flags)
+    messages: Vec<(MessageType, Vec<u8>, u8, u16)>, // (type, data, msg_flags, creation order)
+    /// Attribute creation order tracked and indexed.
+    attr_order: bool,
 }
 
 impl ObjectHeaderWriter {
@@ -22,17 +29,33 @@ impl ObjectHeaderWriter {
     pub fn new() -> Self {
         Self {
             messages: Vec::new(),
+            attr_order: false,
         }
     }
 
+    /// Track and index attribute creation order, as libhdf5 does for an
+    /// object created with `H5P_CRT_ORDER_TRACKED | H5P_CRT_ORDER_INDEXED`
+    /// (h5py's `track_order=True`): the header's flags say so, and every
+    /// message carries a creation order (an attribute's own; 0 for the
+    /// others). libhdf5 reads the setting back from these flags.
+    pub fn track_attr_order(&mut self) {
+        self.attr_order = true;
+    }
+
     /// Add a message to the header with default flags (0).
     pub fn add_message(&mut self, msg_type: MessageType, data: Vec<u8>) {
-        self.messages.push((msg_type, data, 0));
+        self.messages.push((msg_type, data, 0, 0));
     }
 
     /// Add a message with specific flags.
     pub fn add_message_with_flags(&mut self, msg_type: MessageType, data: Vec<u8>, flags: u8) {
-        self.messages.push((msg_type, data, flags));
+        self.messages.push((msg_type, data, flags, 0));
+    }
+
+    /// Add a message with its creation order, which is written only when
+    /// attribute creation order is tracked ([`Self::track_attr_order`]).
+    pub fn add_message_with_order(&mut self, msg_type: MessageType, data: Vec<u8>, order: u16) {
+        self.messages.push((msg_type, data, 0, order));
     }
 
     /// Serialize the complete v2 object header (OHDR + messages + checksum).
@@ -41,10 +64,10 @@ impl ObjectHeaderWriter {
     /// than [`MAX_MESSAGE_SIZE`] (e.g. an attribute over ~64 KiB, which would
     /// need dense attribute storage), rather than writing a corrupt header.
     pub fn serialize(&self) -> Result<Vec<u8>, FormatError> {
-        if let Some((msg_type, data, _)) = self
+        if let Some((msg_type, data, _, _)) = self
             .messages
             .iter()
-            .find(|(_, data, _)| data.len() > MAX_MESSAGE_SIZE)
+            .find(|(_, data, _, _)| data.len() > MAX_MESSAGE_SIZE)
         {
             return Err(FormatError::SerializationError(format!(
                 "{msg_type:?} message is {} bytes; an object header message holds at most \
@@ -52,11 +75,13 @@ impl ObjectHeaderWriter {
                 data.len()
             )));
         }
-        // Calculate total message bytes: each message has type(1) + size(2) + flags(1) + data
+        // Calculate total message bytes: each message has type(1) + size(2) +
+        // flags(1) [+ creation order(2)] + data
+        let msg_header = if self.attr_order { 6 } else { 4 };
         let msg_bytes_total: usize = self
             .messages
             .iter()
-            .map(|(_, data, _)| 4 + data.len())
+            .map(|(_, data, _, _)| msg_header + data.len())
             .sum();
 
         // Determine chunk size field width based on msg_bytes_total
@@ -68,6 +93,12 @@ impl ObjectHeaderWriter {
             (0x02u8, 4)
         };
 
+        let flags = if self.attr_order {
+            flags | OHDR_ATTR_CRT_ORDER_TRACKED | OHDR_ATTR_CRT_ORDER_INDEXED
+        } else {
+            flags
+        };
+
         let mut buf = Vec::new();
 
         // OHDR signature
@@ -85,7 +116,7 @@ impl ObjectHeaderWriter {
         }
 
         // Messages
-        for (msg_type, data, msg_flags) in &self.messages {
+        for (msg_type, data, msg_flags, order) in &self.messages {
             let type_id = msg_type.to_u16();
             assert!(
                 type_id <= 255,
@@ -94,6 +125,9 @@ impl ObjectHeaderWriter {
             buf.push(type_id as u8); // type (1 byte in v2)
             buf.extend_from_slice(&(data.len() as u16).to_le_bytes()); // size (2 bytes)
             buf.push(*msg_flags); // flags
+            if self.attr_order {
+                buf.extend_from_slice(&order.to_le_bytes()); // creation order
+            }
             buf.extend_from_slice(data);
         }
 
@@ -193,6 +227,21 @@ mod tests {
         assert_eq!(hdr.messages.len(), 0);
     }
 
+    #[test]
+    fn tracked_attribute_order_is_in_the_flags_and_every_message() {
+        let mut writer = ObjectHeaderWriter::new();
+        writer.track_attr_order();
+        writer.add_message(MessageType::Dataspace, vec![1, 2, 3, 4]);
+        writer.add_message_with_order(MessageType::Attribute, vec![5, 6], 7);
+        let bytes = writer.serialize().unwrap();
+        assert_eq!(bytes[5] & 0x0C, 0x0C);
+        let hdr = ObjectHeader::parse(&bytes, 0, 8, 8).unwrap();
+        assert_eq!(hdr.messages.len(), 2);
+        assert_eq!(hdr.messages[0].creation_order, Some(0));
+        assert_eq!(hdr.messages[1].creation_order, Some(7));
+        assert_eq!(hdr.messages[1].data, vec![5, 6]);
+    }
+
     #[test]
     fn two_messages_roundtrip() {
         let mut writer = ObjectHeaderWriter::new();

crates/clawhdf5-format/src/parallel_read.rs

@@ -12,6 +12,22 @@ use crate::error::FormatError;
 use crate::filter_pipeline::FilterPipeline;
 use crate::filters::decompress_chunk_exact;
 use crate::lane_partition::{self, LaneStats, PartitionStats};
+use crate::storage::{ExtentReq, Storage, for_each_extent_batch};
+
+/// The extents of `chunks`' stored bytes (see
+/// [`crate::chunked_read::chunk_req`]), fetched batch by batch with
+/// [`for_each_extent_batch`] when the file is not in memory (each chunk's
+/// bounds error is reported when that chunk is decoded, as before).
+fn chunk_reqs(
+    chunks: &[ChunkInfo],
+    pipeline: Option<&FilterPipeline>,
+    chunk_total_bytes: usize,
+) -> Vec<ExtentReq> {
+    chunks
+        .iter()
+        .map(|c| crate::chunked_read::chunk_req(c, pipeline, chunk_total_bytes, true))
+        .collect()
+}
 
 /// Threshold: only use parallel decompression when chunk count exceeds this.
 const PARALLEL_THRESHOLD: usize = 4;
@@ -41,6 +57,132 @@ pub fn pool_can_parallelise() -> bool {
     rayon::current_num_threads() > 1
 }
 
+/// How many rayon workers [`run_with_helpers`] should ask to help with
+/// `items` work items, given that the calling thread works too: the pool's
+/// other threads (all of them when the caller is not one), at most one per
+/// item beyond the caller's first.
+pub(crate) fn helper_count(items: usize) -> usize {
+    let pool = rayon::current_num_threads();
+    // A one-thread pool means "decode on the calling thread" (the setting
+    // benchmarks use to compare with h5py, where each call decodes on its
+    // caller): no helper, so one read never uses two cores.
+    if pool <= 1 {
+        return 0;
+    }
+    let others = if rayon::current_thread_index().is_some() {
+        pool.saturating_sub(1)
+    } else {
+        pool
+    };
+    others.min(items.saturating_sub(1))
+}
+
+/// Run `body` on the calling thread and on up to `helpers` rayon workers at
+/// once, returning when the caller's call has finished and every worker that
+/// started one has too. `body` shares its work out itself (typically by
+/// claiming items from an atomic counter until none are left).
+///
+/// The caller never waits for a worker to *become* free: helpers are queued
+/// on the pool, and one that only gets to run after the caller has finished
+/// returns without calling `body`. So a busy or small pool can only fail to
+/// speed a read up, never hold it back — with `par_iter`, the calling thread
+/// (not a pool worker) handed all the work to the pool and slept, and N
+/// threads reading through a 2-worker pool decoded on 2 cores.
+///
+/// A panic in `body`, on any thread, is resumed on the caller once every
+/// helper that started has stopped.
+pub(crate) fn run_with_helpers(helpers: usize, body: &(dyn Fn() + Sync)) {
+    use std::panic::{AssertUnwindSafe, catch_unwind, resume_unwind};
+    use std::sync::{Arc, Condvar, Mutex, PoisonError};
+
+    if helpers == 0 {
+        body();
+        return;
+    }
+
+    type Body = dyn Fn() + Sync + 'static;
+    struct Shared {
+        /// `body`, its lifetime erased. Only dereferenced by a helper that
+        /// registered in `state` while it was open (see below).
+        body: *const Body,
+        /// (closed, helpers inside `body`).
+        state: Mutex<(bool, usize)>,
+        idle: Condvar,
+        panic: Mutex<Option<Box<dyn core::any::Any + Send>>>,
+    }
+    // SAFETY: `body` points to a `Sync` closure, so calling it from other
+    // threads is allowed; the pointer is only used under the protocol below,
+    // which keeps it from outliving the closure.
+    unsafe impl Send for Shared {}
+    unsafe impl Sync for Shared {}
+
+    fn help(shared: &Shared) {
+        {
+            let mut state = shared.state.lock().unwrap_or_else(PoisonError::into_inner);
+            if state.0 {
+                return;
+            }
+            state.1 += 1;
+        }
+        // SAFETY: registered while open, so the caller of `run_with_helpers`
+        // is still inside it (it closes, then waits until no helper is
+        // registered, before returning), and `body` is alive.
+        let body = unsafe { &*shared.body };
+        if let Err(payload) = catch_unwind(AssertUnwindSafe(body)) {
+            shared
+                .panic
+                .lock()
+                .unwrap_or_else(PoisonError::into_inner)
+                .get_or_insert(payload);
+        }
+        let mut state = shared.state.lock().unwrap_or_else(PoisonError::into_inner);
+        state.1 -= 1;
+        if state.1 == 0 {
+            shared.idle.notify_all();
+        }
+    }
+
+    let body_ptr: *const (dyn Fn() + Sync + '_) = body;
+    // SAFETY: only the lifetime changes (same fat-pointer layout). The
+    // pointer is dereferenced only while this function is running: see
+    // `help` and the wait below.
+    let body_ptr: *const Body = unsafe { core::mem::transmute(body_ptr) };
+    let shared = Arc::new(Shared {
+        body: body_ptr,
+        state: Mutex::new((false, 0)),
+        idle: Condvar::new(),
+        panic: Mutex::new(None),
+    });
+    for _ in 0..helpers {
+        let shared = Arc::clone(&shared);
+        rayon::spawn(move || help(&shared));
+    }
+    let caller = catch_unwind(AssertUnwindSafe(body));
+    {
+        // Close, then wait for the helpers inside `body`; later ones return
+        // at once. This must happen even if `body` panicked on this thread.
+        let mut state = shared.state.lock().unwrap_or_else(PoisonError::into_inner);
+        state.0 = true;
+        while state.1 > 0 {
+            state = shared
+                .idle
+                .wait(state)
+                .unwrap_or_else(PoisonError::into_inner);
+        }
+    }
+    if let Err(payload) = caller {
+        resume_unwind(payload);
+    }
+    let helper_panic = shared
+        .panic
+        .lock()
+        .unwrap_or_else(PoisonError::into_inner)
+        .take();
+    if let Some(payload) = helper_panic {
+        resume_unwind(payload);
+    }
+}
+
 /// Decompress chunks in parallel using lane-partitioned assignment.
 ///
 /// Instead of naive `par_iter`, chunks are deterministically assigned to lanes
@@ -63,6 +205,27 @@ pub fn decompress_chunks_lane_partitioned(
     element_size: u32,
     seed: u64,
     num_lanes: Option<usize>,
+) -> Result<(Vec<Vec<u8>>, PartitionStats), FormatError> {
+    decompress_chunks_lane_partitioned_in(
+        file_data,
+        chunks,
+        pipeline,
+        chunk_total_bytes,
+        element_size,
+        seed,
+        num_lanes,
+    )
+}
+
+/// [`decompress_chunks_lane_partitioned`] over any [`Storage`].
+pub fn decompress_chunks_lane_partitioned_in<S: Storage + ?Sized>(
+    file_data: &S,
+    chunks: &[ChunkInfo],
+    pipeline: &FilterPipeline,
+    chunk_total_bytes: usize,
+    element_size: u32,
+    seed: u64,
+    num_lanes: Option<usize>,
 ) -> Result<(Vec<Vec<u8>>, PartitionStats), FormatError> {
     use rayon::prelude::*;
 
@@ -72,31 +235,28 @@ pub fn decompress_chunks_lane_partitioned(
             .unwrap_or(1)
     });
 
-    let assignments = lane_partition::partition_chunks(chunks.len(), lanes, seed);
+    let reqs = chunk_reqs(chunks, Some(pipeline), chunk_total_bytes);
-    let num_lanes = assignments.len();
+    let mut ordered: Vec<Vec<u8>> = Vec::with_capacity(chunks.len());
-
+    let mut partition_stats = PartitionStats::new(0);
+    partition_stats.total_chunks = chunks.len();
+    // Each batch of fetched chunks is partitioned into lanes and decoded
+    // before the next batch is fetched (with the file in memory there is
+    // one batch: all the chunks).
+    for_each_extent_batch(file_data, &reqs, |batch, raw_bytes| {
+        let assignments = lane_partition::partition_chunks(batch.len(), lanes, seed);
         // Each lane processes its assigned chunks and returns results + stats.
-    let lane_results: Result<Vec<(Vec<DecompressedChunk>, LaneStats)>, FormatError> = assignments
+        let lane_results: Result<Vec<(Vec<DecompressedChunk>, LaneStats)>, FormatError> =
+            assignments
                 .into_par_iter()
                 .map(|indices| {
                     let mut results = Vec::with_capacity(indices.len());
                     let mut stats = LaneStats::default();
 
-            for &index in &indices {
+                    for &local in &indices {
+                        let index = batch.start + local;
                         let chunk_info = &chunks[index];
-                let c_addr = chunk_info.address as usize;
+                        let size = crate::addr::saturating_usize(chunk_info.chunk_size);
-                let size = chunk_info.chunk_size as usize;
+                        let raw_chunk = raw_bytes.get(index, &reqs[index])?;
-
-                if c_addr
-                    .checked_add(size)
-                    .is_none_or(|end| end > file_data.len())
-                {
-                    return Err(FormatError::UnexpectedEof {
-                        expected: c_addr.saturating_add(size),
-                        available: file_data.len(),
-                    });
-                }
-                let raw_chunk = &file_data[c_addr..c_addr + size];
 
                         let decompressed = decompress_chunk_exact(
                             raw_chunk,
@@ -120,14 +280,19 @@ pub fn decompress_chunks_lane_partitioned(
                     Ok((results, stats))
                 })
                 .collect();
-
         let lane_results = lane_results?;
 
         // Aggregate stats
-    let mut partition_stats = PartitionStats::new(num_lanes);
+        if partition_stats.per_lane.len() < lane_results.len() {
-    partition_stats.total_chunks = chunks.len();
+            partition_stats
-    for (lane_idx, (_, stats)) in lane_results.iter().enumerate() {
+                .per_lane
-        partition_stats.per_lane[lane_idx] = stats.clone();
+                .resize_with(lane_results.len(), LaneStats::default);
+            partition_stats.num_lanes = lane_results.len();
+        }
+        for (lane, (_, stats)) in partition_stats.per_lane.iter_mut().zip(&lane_results) {
+            lane.chunks_processed += stats.chunks_processed;
+            lane.compressed_bytes += stats.compressed_bytes;
+            lane.decompressed_bytes += stats.decompressed_bytes;
         }
 
         // Flatten and sort by original index to restore order
@@ -136,8 +301,9 @@ pub fn decompress_chunks_lane_partitioned(
             .flat_map(|(chunks, _)| chunks)
             .collect();
         all_chunks.sort_by_key(|dc| dc.index);
-
+        ordered.extend(all_chunks.into_iter().map(|dc| dc.data));
-    let ordered = all_chunks.into_iter().map(|dc| dc.data).collect();
+        Ok(())
+    })?;
     Ok((ordered, partition_stats))
 }
 
@@ -155,25 +321,29 @@ pub fn decompress_chunks_parallel(
     pipeline: &FilterPipeline,
     chunk_total_bytes: usize,
     element_size: u32,
+) -> Result<Vec<Vec<u8>>, FormatError> {
+    decompress_chunks_parallel_in(file_data, chunks, pipeline, chunk_total_bytes, element_size)
+}
+
+/// [`decompress_chunks_parallel`] over any [`Storage`].
+pub fn decompress_chunks_parallel_in<S: Storage + ?Sized>(
+    file_data: &S,
+    chunks: &[ChunkInfo],
+    pipeline: &FilterPipeline,
+    chunk_total_bytes: usize,
+    element_size: u32,
 ) -> Result<Vec<Vec<u8>>, FormatError> {
     use rayon::prelude::*;
 
-    let results: Result<Vec<DecompressedChunk>, FormatError> = chunks
+    let reqs = chunk_reqs(chunks, Some(pipeline), chunk_total_bytes);
-        .par_iter()
+    let mut ordered: Vec<Vec<u8>> = Vec::with_capacity(chunks.len());
-        .enumerate()
+    for_each_extent_batch(file_data, &reqs, |batch, raw_bytes| {
-        .map(|(index, chunk_info)| {
+        let results: Result<Vec<DecompressedChunk>, FormatError> = batch
-            let c_addr = chunk_info.address as usize;
+            .clone()
-            let size = chunk_info.chunk_size as usize;
+            .into_par_iter()
-            if c_addr
+            .map(|index| {
-                .checked_add(size)
+                let chunk_info = &chunks[index];
-                .is_none_or(|end| end > file_data.len())
+                let raw_chunk = raw_bytes.get(index, &reqs[index])?;
-            {
-                return Err(FormatError::UnexpectedEof {
-                    expected: c_addr.saturating_add(size),
-                    available: file_data.len(),
-                });
-            }
-            let raw_chunk = &file_data[c_addr..c_addr + size];
 
                 let decompressed = decompress_chunk_exact(
                     raw_chunk,
@@ -193,7 +363,10 @@ pub fn decompress_chunks_parallel(
 
         let mut result_vec = results?;
         result_vec.sort_by_key(|dc| dc.index);
-    Ok(result_vec.into_iter().map(|dc| dc.data).collect())
+        ordered.extend(result_vec.into_iter().map(|dc| dc.data));
+        Ok(())
+    })?;
+    Ok(ordered)
 }
 
 /// Decompress chunks sequentially (fallback when parallel is not warranted).
@@ -204,20 +377,23 @@ pub fn decompress_chunks_sequential(
     chunk_total_bytes: usize,
     element_size: u32,
 ) -> Result<Vec<Vec<u8>>, FormatError> {
+    decompress_chunks_sequential_in(file_data, chunks, pipeline, chunk_total_bytes, element_size)
+}
+
+/// [`decompress_chunks_sequential`] over any [`Storage`].
+pub fn decompress_chunks_sequential_in<S: Storage + ?Sized>(
+    file_data: &S,
+    chunks: &[ChunkInfo],
+    pipeline: Option<&FilterPipeline>,
+    chunk_total_bytes: usize,
+    element_size: u32,
+) -> Result<Vec<Vec<u8>>, FormatError> {
+    let reqs = chunk_reqs(chunks, pipeline, chunk_total_bytes);
     let mut result = Vec::with_capacity(chunks.len());
-    for chunk_info in chunks {
+    for_each_extent_batch(file_data, &reqs, |batch, raw_bytes| {
-        let c_addr = chunk_info.address as usize;
+        for i in batch {
-        let size = chunk_info.chunk_size as usize;
+            let chunk_info = &chunks[i];
-        if c_addr
+            let raw_chunk = raw_bytes.get(i, &reqs[i])?;
-            .checked_add(size)
-            .is_none_or(|end| end > file_data.len())
-        {
-            return Err(FormatError::UnexpectedEof {
-                expected: c_addr.saturating_add(size),
-                available: file_data.len(),
-            });
-        }
-        let raw_chunk = &file_data[c_addr..c_addr + size];
 
             let decompressed = if let Some(pl) = pipeline {
                 decompress_chunk_exact(
@@ -233,6 +409,8 @@ pub fn decompress_chunks_sequential(
             };
             result.push(decompressed);
         }
+        Ok(())
+    })?;
     Ok(result)
 }
 
@@ -248,7 +426,7 @@ mod tests {
         for i in 0..8u64 {
             let len = if short && i == 5 { 16 } else { 32 };
             infos.push(ChunkInfo {
-                chunk_size: len as u32,
+                chunk_size: len as u64,
                 filter_mask: 0,
                 offsets: vec![i * 8],
                 address: file.len() as u64,
@@ -258,6 +436,60 @@ mod tests {
         (file, infos)
     }
 
+    /// Every item is processed exactly once, whatever mix of caller and
+    /// helpers ends up doing it.
+    #[test]
+    fn run_with_helpers_shares_all_work() {
+        use core::sync::atomic::{AtomicUsize, Ordering};
+        for helpers in [0, 1, 3, 16] {
+            let n = 1000;
+            let next = AtomicUsize::new(0);
+            let done: Vec<AtomicUsize> = (0..n).map(|_| AtomicUsize::new(0)).collect();
+            run_with_helpers(helpers, &|| {
+                loop {
+                    let i = next.fetch_add(1, Ordering::Relaxed);
+                    if i >= n {
+                        break;
+                    }
+                    done[i].fetch_add(1, Ordering::Relaxed);
+                }
+            });
+            assert!(done.iter().all(|d| d.load(Ordering::Relaxed) == 1));
+        }
+    }
+
+    /// A panic in the shared body reaches the caller whichever thread it
+    /// happened on, and only after the helpers inside the body have left it
+    /// (they borrow the caller's stack).
+    #[test]
+    fn run_with_helpers_propagates_panics() {
+        use core::sync::atomic::{AtomicUsize, Ordering};
+        use std::panic::{AssertUnwindSafe, catch_unwind};
+        let caller = std::thread::current().id();
+        for panic_on_caller in [true, false] {
+            let inside = AtomicUsize::new(0);
+            let calls = AtomicUsize::new(0);
+            let result = catch_unwind(AssertUnwindSafe(|| {
+                run_with_helpers(4, &|| {
+                    inside.fetch_add(1, Ordering::SeqCst);
+                    calls.fetch_add(1, Ordering::SeqCst);
+                    let on_caller = std::thread::current().id() == caller;
+                    std::thread::sleep(std::time::Duration::from_millis(20));
+                    inside.fetch_sub(1, Ordering::SeqCst);
+                    if on_caller == panic_on_caller {
+                        panic!("boom");
+                    }
+                });
+            }));
+            // A helper may never have run (the pool was slow to start it),
+            // in which case nothing panicked when `panic_on_caller` is false.
+            if panic_on_caller || calls.load(Ordering::SeqCst) > 1 {
+                assert!(result.is_err());
+            }
+            assert_eq!(inside.load(Ordering::SeqCst), 0);
+        }
+    }
+
     /// Every parallel decoder refuses a chunk that decodes short, naming it.
     #[test]
     fn short_decoded_chunk_is_an_error() {

crates/clawhdf5-format/src/partial_read.rs

@@ -18,14 +18,15 @@ use alloc::{format, vec, vec::Vec};
 #[cfg(feature = "std")]
 use std::string as alloc_or_std;
 
-use crate::chunked_read::{alloc_output, checked_byte_len, list_chunks};
+use crate::chunked_read::{alloc_output, checked_byte_len, list_chunks_for_read_in};
 use crate::data_layout::DataLayout;
 use crate::data_read::extract_selection_from_buffer;
 use crate::dataspace::Dataspace;
 use crate::error::FormatError;
 use crate::filter_pipeline::FilterPipeline;
-use crate::filters::{all_filters_skipped, decompress_chunk_exact};
+use crate::filters::{all_filters_skipped, decompress_chunk_exact_with};
 use crate::selection::Selection;
+use crate::storage::{ExtentReq, Storage, for_each_extent_batch};
 
 /// The smallest axis-aligned box containing every selected element, as
 /// `(start, extent)` per dimension. `None` when there is nothing to gain or
@@ -203,7 +204,12 @@ fn copy_overlap(
     };
     let (src_strides, out_strides) = (strides(src_shape), strides(box_extent));
     let last = rank - 1;
-    let run = ((hi[last] - lo[last]) as usize) * elem_size;
+    // Byte offsets into the in-memory buffers; one that does not fit `usize`
+    // (a 32-bit target) is out of both buffers, like one past their ends.
+    let bytes = |elements: u64| usize::try_from(elements).ok()?.checked_mul(elem_size);
+    let Some(run) = bytes(hi[last] - lo[last]) else {
+        return;
+    };
 
     let mut idx = lo.clone();
     loop {
@@ -213,8 +219,12 @@ fn copy_overlap(
         let out_at: u64 = (0..rank)
             .map(|d| (idx[d] - box_start[d]) * out_strides[d])
             .sum();
-        let (s, o) = (src_at as usize * elem_size, out_at as usize * elem_size);
+        if let (Some(s), Some(o)) = (bytes(src_at), bytes(out_at))
-        if let (Some(from), Some(to)) = (src.get(s..s + run), out.get_mut(o..o + run)) {
+            && let (Some(from), Some(to)) = (
+                src.get(s..s.saturating_add(run)),
+                out.get_mut(o..o.saturating_add(run)),
+            )
+        {
             to.copy_from_slice(from);
         }
         // Advance over every dimension but the last.
@@ -233,6 +243,58 @@ fn copy_overlap(
     }
 }
 
+/// Copy the part of unfiltered chunk `chunk` (`chunk_bytes` long, shape
+/// `chunk_shape`) that overlaps the box into `out`, reading only the runs of
+/// the overlap from the file. The whole chunk must still lie inside the
+/// file, as it must when it is fetched whole.
+#[allow(clippy::too_many_arguments)]
+fn read_unfiltered_overlap<S: Storage + ?Sized>(
+    file_data: &S,
+    chunk: &crate::chunked_read::ChunkInfo,
+    chunk_bytes: usize,
+    chunk_shape: &[u64],
+    elem_size: usize,
+    out: &mut [u8],
+    box_start: &[u64],
+    box_extent: &[u64],
+) -> Result<(), FormatError> {
+    let rank = chunk_shape.len();
+    let origin = &chunk.offsets[..rank];
+    let (lo, extent): (Vec<u64>, Vec<u64>) = (0..rank)
+        .map(|d| {
+            let lo = origin[d].max(box_start[d]);
+            let hi = origin[d]
+                .saturating_add(chunk_shape[d])
+                .min(box_start[d] + box_extent[d]);
+            (lo, hi.saturating_sub(lo))
+        })
+        .unzip();
+    let file_len = crate::storage::len_usize(file_data);
+    let base = crate::addr::to_usize(chunk.address)?;
+    if base > file_len || chunk_bytes > file_len - base {
+        return Err(FormatError::UnexpectedEof {
+            expected: base.saturating_add(chunk_bytes),
+            available: file_len,
+        });
+    }
+    let overlap = Selection::Hyperslab {
+        start: lo.iter().zip(origin).map(|(l, o)| l - o).collect(),
+        stride: vec![1; rank],
+        count: extent.clone(),
+        block: vec![1; rank],
+    };
+    let rows = crate::gather::gather_storage(
+        file_data,
+        chunk.address,
+        chunk_bytes,
+        chunk_shape,
+        elem_size,
+        &overlap,
+    )?;
+    copy_overlap(&rows, &lo, &extent, out, box_start, box_extent, elem_size);
+    Ok(())
+}
+
 /// Read `selection` without materialising the whole dataset, when that is
 /// possible and worthwhile. `Ok(None)` means "use the full-read path": an
 /// `All`/`None`/invalid selection, a layout this doesn't handle (compact,
@@ -247,9 +309,69 @@ pub fn read_selection(
     offset_size: u8,
     length_size: u8,
     selection: &Selection,
+) -> Result<Option<Vec<u8>>, FormatError> {
+    read_selection_in(
+        file_data,
+        layout,
+        dataspace,
+        elem_size,
+        pipeline,
+        offset_size,
+        length_size,
+        selection,
+    )
+}
+
+/// [`read_selection`] over any [`Storage`].
+#[allow(clippy::too_many_arguments)]
+pub fn read_selection_in<S: Storage + ?Sized>(
+    file_data: &S,
+    layout: &DataLayout,
+    dataspace: &Dataspace,
+    elem_size: usize,
+    pipeline: Option<&FilterPipeline>,
+    offset_size: u8,
+    length_size: u8,
+    selection: &Selection,
+) -> Result<Option<Vec<u8>>, FormatError> {
+    read_selection_filled_in(
+        file_data,
+        layout,
+        dataspace,
+        elem_size,
+        pipeline,
+        offset_size,
+        length_size,
+        selection,
+        None,
+    )
+}
+
+/// [`read_selection_in`] for a dataset whose fill value is `fill` (one
+/// element's bytes; `None` or all zeros is the default fill): the elements
+/// of a chunked dataset's selection that lie in chunks never written read as
+/// `fill`, as they do in a full read. Only the chunks the selection's
+/// bounding box overlaps are read, so a selection of a few elements of a
+/// dataset whose chunks are 4 GiB or more costs one decoded chunk (a
+/// filtered chunk has to be decoded whole) or, unfiltered, only the bytes
+/// it selects.
+#[allow(clippy::too_many_arguments)]
+pub fn read_selection_filled_in<S: Storage + ?Sized>(
+    file_data: &S,
+    layout: &DataLayout,
+    dataspace: &Dataspace,
+    elem_size: usize,
+    pipeline: Option<&FilterPipeline>,
+    offset_size: u8,
+    length_size: u8,
+    selection: &Selection,
+    fill: Option<&[u8]>,
 ) -> Result<Option<Vec<u8>>, FormatError> {
     let dims = &dataspace.dimensions;
-    if dims.is_empty() || elem_size == 0 {
+    // A fill value that is not one element's bytes is the full path's to
+    // interpret.
+    let odd_fill = fill.is_some_and(|f| !f.is_empty() && f.len() != elem_size);
+    if dims.is_empty() || elem_size == 0 || odd_fill {
         return Ok(None);
     }
     let total = dataspace.checked_num_elements()?;
@@ -267,15 +389,34 @@ pub fn read_selection(
         validate(selection, dims)?;
         let base = usize::try_from(*address)
             .map_err(|_| FormatError::Overflow("data address exceeds usize".into()))?;
-        let data = file_data
+        let file_len = crate::storage::len_usize(file_data);
+        let eof = FormatError::UnexpectedEof {
+            expected: base,
+            available: file_len,
+        };
+        if let Some(all) = file_data.as_contiguous() {
+            let data = all
                 .get(base..)
                 .and_then(|d| d.get(..checked_byte_len(total, elem_size).ok()?))
-            .ok_or(FormatError::UnexpectedEof {
+                .ok_or(eof)?;
-                expected: base,
-                available: file_data.len(),
-            })?;
             return crate::gather::gather::<u8>(data, dims, elem_size, selection).map(Some);
         }
+        // Not in memory: the same bounds check, then only the selected runs
+        // are read.
+        let len = checked_byte_len(total, elem_size)
+            .ok()
+            .filter(|&len| base <= file_len && len <= file_len - base)
+            .ok_or(eof)?;
+        return crate::gather::gather_storage(
+            file_data,
+            base as u64,
+            len,
+            dims,
+            elem_size,
+            selection,
+        )
+        .map(Some);
+    }
     let Some((box_start, box_extent)) = bounding_box(selection, dims) else {
         return Ok(None);
     };
@@ -288,58 +429,96 @@ pub fn read_selection(
         return Ok(None);
     }
     let mut boxed = alloc_output(checked_byte_len(box_elements, elem_size)?)?;
+    if let Some(fill) = fill.filter(|f| <[u8]>::len(f) == elem_size && f.iter().any(|&b| b != 0)) {
+        for element in boxed.chunks_exact_mut(elem_size) {
+            element.copy_from_slice(fill);
+        }
+    }
 
     match layout {
+        // No chunk was ever written: every element is the fill value.
+        DataLayout::Chunked {
+            btree_address: None,
+            ..
+        } if fill.is_some() => {}
         DataLayout::Chunked {
             btree_address: Some(_),
             ..
         } => {
-            let (chunks, chunk_dims) = list_chunks(
+            let (chunks, chunk_dims) = list_chunks_for_read_in(
                 file_data,
                 layout,
                 dataspace,
                 elem_size,
+                pipeline,
                 offset_size,
                 length_size,
             )?;
             let rank = dims.len();
             let chunk_shape: Vec<u64> = chunk_dims.iter().map(|&d| d as u64).collect();
             let chunk_bytes = crate::chunked_read::checked_chunk_byte_len(&chunk_dims, elem_size)?;
-            for chunk in &chunks {
+            // The chunks overlapping the box, in index order.
+            let wanted: Vec<&crate::chunked_read::ChunkInfo> = chunks
+                .iter()
+                .filter(|chunk| {
                     if chunk.offsets.len() < rank || chunk.address == u64::MAX {
-                    continue;
+                        return false;
                     }
                     let origin = &chunk.offsets[..rank];
-                let overlaps = (0..rank).all(|d| {
+                    (0..rank).all(|d| {
                         origin[d] < box_start[d] + box_extent[d]
                             && origin[d].saturating_add(chunk_shape[d]) > box_start[d]
+                    })
+                })
+                .collect();
+            // An unfiltered chunk of a file that is not in memory: fetch
+            // only the rows the box needs, not the whole chunk (which may be
+            // 4 GiB or more).
+            let (direct, wanted): (Vec<_>, Vec<_>) = wanted.into_iter().partition(|c| {
+                file_data.as_contiguous().is_none()
+                    && pipeline.is_none_or(|pl| all_filters_skipped(pl, c.filter_mask))
             });
-                if !overlaps {
+            for chunk in direct {
-                    continue;
+                read_unfiltered_overlap(
+                    file_data,
+                    chunk,
+                    chunk_bytes,
+                    &chunk_shape,
+                    elem_size,
+                    &mut boxed,
+                    &box_start,
+                    &box_extent,
+                )?;
             }
-                let at = usize::try_from(chunk.address)
+            // Their stored bytes, batch by batch when the file is not in
-                    .map_err(|_| FormatError::Overflow("chunk address exceeds usize".into()))?;
+            // memory; each batch's chunks are decoded into this thread's
-                let raw = at
+            // reusable buffers before the next batch is fetched.
-                    .checked_add(chunk.chunk_size as usize)
+            let reqs: Vec<ExtentReq> = wanted
-                    .and_then(|end| file_data.get(at..end))
+                .iter()
-                    .ok_or(FormatError::UnexpectedEof {
+                .map(|c| crate::chunked_read::chunk_req(c, pipeline, chunk_bytes, true))
-                        expected: at.saturating_add(chunk.chunk_size as usize),
+                .collect();
-                        available: file_data.len(),
+            for_each_extent_batch(file_data, &reqs, |batch, raw_bytes| {
+                crate::chunked_read::with_scratch(|scratch| -> Result<(), FormatError> {
+                    for i in batch {
+                        let chunk = wanted[i];
+                        let origin = &chunk.offsets[..rank];
+                        usize::try_from(chunk.address).map_err(|_| {
+                            FormatError::Overflow("chunk address exceeds usize".into())
                         })?;
-                // Mirrors the full-read path: filter-mask bit i set means
+                        let raw = raw_bytes.get(i, &reqs[i])?;
-                // filter i was not applied to this chunk.
+                        // Mirrors the full-read path: filter-mask bit i set
-                let decoded;
+                        // means filter i was not applied to this chunk.
                         let data: &[u8] = match pipeline {
                             Some(pl) if !all_filters_skipped(pl, chunk.filter_mask) => {
-                        decoded = decompress_chunk_exact(
+                                decompress_chunk_exact_with(
                                     raw,
                                     pl,
                                     chunk_bytes,
                                     elem_size as u32,
                                     chunk.filter_mask,
                                     &chunk.offsets[..rank],
-                        )?;
+                                    scratch,
-                        &decoded
+                                )?
                             }
                             _ => raw,
                         };
@@ -353,6 +532,9 @@ pub fn read_selection(
                             elem_size,
                         );
                     }
+                    Ok(())
+                })
+            })?;
         }
         _ => return Ok(None),
     }

crates/clawhdf5-format/src/provenance.rs

@@ -13,7 +13,6 @@ use sha2::{Digest, Sha256};
 
 use crate::attribute::AttributeMessage;
 use crate::data_layout::DataLayout;
-use crate::data_read::read_raw_data;
 use crate::dataspace::Dataspace;
 use crate::datatype::Datatype;
 use crate::error::FormatError;
@@ -128,10 +127,20 @@ pub fn verify_dataset(
     header: &ObjectHeader,
     offset_size: u8,
     length_size: u8,
+) -> Result<VerifyResult, FormatError> {
+    verify_dataset_in(file_data, header, offset_size, length_size)
+}
+
+/// [`verify_dataset`] over any [`Storage`](crate::storage::Storage).
+pub fn verify_dataset_in<S: crate::storage::Storage + ?Sized>(
+    file_data: &S,
+    header: &ObjectHeader,
+    offset_size: u8,
+    length_size: u8,
 ) -> Result<VerifyResult, FormatError> {
     // 1. Extract all attributes (compact + dense).
     let attrs =
-        crate::attribute::extract_attributes_full(file_data, header, offset_size, length_size)?;
+        crate::attribute::extract_attributes_full_in(file_data, header, offset_size, length_size)?;
 
     // 2. Find the stored hash.
     let stored_hash = attrs
@@ -174,7 +183,7 @@ pub fn verify_dataset(
         .transpose()?;
 
     let raw = match &dl {
-        DataLayout::Chunked { .. } => crate::chunked_read::read_chunked_data(
+        DataLayout::Chunked { .. } => crate::chunked_read::read_chunked_data_in(
             file_data,
             &dl,
             &ds,
@@ -183,7 +192,7 @@ pub fn verify_dataset(
             offset_size,
             length_size,
         )?,
-        _ => read_raw_data(file_data, &dl, &ds, &dt)?,
+        _ => crate::data_read::read_raw_data_in(file_data, &dl, &ds, &dt)?,
     };
 
     // 4. Compare.

crates/clawhdf5-format/src/selection.rs

@@ -19,6 +19,7 @@ use alloc::{vec, vec::Vec};
 
 use core::ops::Range;
 
+use crate::addr::to_usize;
 use crate::error::FormatError;
 
 /// A selection describing which elements of a dataset to access.
@@ -562,7 +563,7 @@ fn decode_hyperslab(r: &mut SelReader, version: u64) -> Result<SerializedSelecti
     if !matches!(enc_size, 2 | 4 | 8) {
         return Err(sel_err("unsupported hyperslab coordinate encoding size"));
     }
-    let rank = r.uint(4)? as usize;
+    let rank = to_usize(r.uint(4)?)?;
     // HDF5 caps dataspace rank at 32 (H5S_MAX_RANK). Reject anything else so a
     // corrupt rank can't drive a huge allocation or read loop.
     if rank == 0 || rank > 32 {
@@ -625,11 +626,11 @@ fn decode_hyperslab(r: &mut SelReader, version: u64) -> Result<SerializedSelecti
         return Err(FormatError::UnexpectedEof {
             expected: r
                 .pos
-                .saturating_add(nblocks.saturating_mul(per_block) as usize),
+                .saturating_add(to_usize(nblocks.saturating_mul(per_block))?),
             available: r.data.len(),
         });
     }
-    let n = nblocks as usize * rank;
+    let n = to_usize(nblocks)? * rank;
     let (mut starts, mut ends) = (Vec::with_capacity(n), Vec::with_capacity(n));
     for _ in 0..nblocks {
         for _ in 0..rank {
@@ -662,7 +663,7 @@ fn blocks_union_coords(
             .filter(|&t| t <= MAX_EXPANDED_POINTS)
             .ok_or_else(|| sel_err("irregular hyperslab selection is too large to expand"))?;
     }
-    let mut out = Vec::with_capacity(total as usize);
+    let mut out = Vec::with_capacity(to_usize(total)?);
     for (s, e) in starts.chunks_exact(rank).zip(ends.chunks_exact(rank)) {
         let mut cur = s.to_vec();
         'block: loop {

crates/clawhdf5-format/src/shared_message.rs

@@ -23,11 +23,12 @@ use alloc::vec::Vec;
 #[cfg(feature = "std")]
 use std::borrow::Cow;
 
-use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records};
+use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records_in};
 use crate::error::FormatError;
 use crate::fractal_heap::FractalHeapHeader;
 use crate::message_type::MessageType;
 use crate::object_header::ObjectHeader;
+use crate::storage::{Storage, Window, read_exact_at};
 
 /// Fractal heap ID length for SOHM entries (fixed at 8 bytes).
 const FHEAP_ID_LEN: usize = 8;
@@ -253,17 +254,31 @@ pub fn parse_sohm_table(
     nindexes: u8,
     offset_size: u8,
 ) -> Result<SohmTable, FormatError> {
-    ensure_len(file_data, table_addr, 4)?;
+    parse_sohm_table_in(file_data, table_addr as u64, nindexes, offset_size)
-    if &file_data[table_addr..table_addr + 4] != b"SMTB" {
+}
+
+/// [`parse_sohm_table`] over any [`Storage`]: one read of the signature,
+/// one of every index entry.
+pub fn parse_sohm_table_in<S: Storage + ?Sized>(
+    file: &S,
+    table_addr: u64,
+    nindexes: u8,
+    offset_size: u8,
+) -> Result<SohmTable, FormatError> {
+    let sig = read_exact_at(file, table_addr, 4)?;
+    if *sig != *b"SMTB" {
         return Err(FormatError::InvalidSohmTableSignature);
     }
-    let mut pos = table_addr + 4;
     let os = offset_size as usize;
     let entry_size = 1 + 1 + 2 + 4 + 2 + 2 + 2 + os + os; // 14 + 2*offset_size
+    // Positions below are relative to the table.
+    let w = Window::read(file, table_addr, 4 + nindexes as usize * entry_size)?;
+    let file_data: &[u8] = &w.bytes;
+    let mut pos = 4;
 
     let mut indexes = Vec::with_capacity(nindexes as usize);
     for _ in 0..nindexes {
-        ensure_len(file_data, pos, entry_size)?;
+        w.ensure(pos, entry_size)?;
         let version = file_data[pos];
         if version != 0 {
             return Err(FormatError::InvalidSohmTableVersion(version));
@@ -369,16 +384,29 @@ pub fn parse_sohm_list(
     num_messages: u16,
     offset_size: u8,
 ) -> Result<Vec<SohmEntry>, FormatError> {
-    ensure_len(file_data, list_addr, 4)?;
+    parse_sohm_list_in(file_data, list_addr as u64, num_messages, offset_size)
-    if &file_data[list_addr..list_addr + 4] != b"SMLI" {
+}
+
+/// [`parse_sohm_list`] over any [`Storage`]: one read of the signature, one
+/// of every entry.
+pub fn parse_sohm_list_in<S: Storage + ?Sized>(
+    file: &S,
+    list_addr: u64,
+    num_messages: u16,
+    offset_size: u8,
+) -> Result<Vec<SohmEntry>, FormatError> {
+    let sig = read_exact_at(file, list_addr, 4)?;
+    if *sig != *b"SMLI" {
         return Err(FormatError::InvalidSohmListSignature);
     }
     let entry_sz = sohm_entry_size(offset_size);
-    let mut pos = list_addr + 4;
+    // Positions below are relative to the list.
+    let w = Window::read(file, list_addr, 4 + num_messages as usize * entry_sz)?;
+    let mut pos = 4;
     let mut entries = Vec::with_capacity(num_messages as usize);
     for _ in 0..num_messages {
-        ensure_len(file_data, pos, entry_sz)?;
+        w.ensure(pos, entry_sz)?;
-        let entry = parse_sohm_entry(&file_data[pos..], offset_size)?;
+        let entry = parse_sohm_entry(&w.bytes[pos..], offset_size)?;
         entries.push(entry);
         pos += entry_sz;
     }
@@ -392,8 +420,18 @@ pub fn parse_sohm_btree_entries(
     offset_size: u8,
     length_size: u8,
 ) -> Result<Vec<SohmEntry>, FormatError> {
-    let header = BTreeV2Header::parse(file_data, btree_addr, offset_size, length_size)?;
+    parse_sohm_btree_entries_in(file_data, btree_addr as u64, offset_size, length_size)
-    let records = collect_btree_v2_records(file_data, &header, offset_size, length_size)?;
+}
+
+/// [`parse_sohm_btree_entries`] over any [`Storage`].
+pub fn parse_sohm_btree_entries_in<S: Storage + ?Sized>(
+    file: &S,
+    btree_addr: u64,
+    offset_size: u8,
+    length_size: u8,
+) -> Result<Vec<SohmEntry>, FormatError> {
+    let header = BTreeV2Header::parse_in(file, btree_addr, offset_size, length_size)?;
+    let records = collect_btree_v2_records_in(file, &header, offset_size, length_size)?;
     let mut entries = Vec::with_capacity(records.len());
     for rec in &records {
         let entry = parse_sohm_entry(&rec.data, offset_size)?;
@@ -413,15 +451,24 @@ pub fn load_sohm_table(
     offset_size: u8,
     length_size: u8,
 ) -> Result<Option<SohmTable>, FormatError> {
-    let sig = crate::signature::find_signature(file_data)?;
+    load_sohm_table_in(file_data, offset_size, length_size)
-    let sb = crate::superblock::Superblock::parse(file_data, sig)?;
+}
+
+/// [`load_sohm_table`] over any [`Storage`].
+pub fn load_sohm_table_in<S: Storage + ?Sized>(
+    file_data: &S,
+    offset_size: u8,
+    length_size: u8,
+) -> Result<Option<SohmTable>, FormatError> {
+    let sig = crate::signature::find_signature_in(file_data)?;
+    let sb = crate::superblock::Superblock::parse_in(file_data, sig)?;
     let Some(ext_addr) = sb
         .superblock_extension_address
         .filter(|&a| !is_undefined(a, offset_size))
     else {
         return Ok(None);
     };
-    let ext = ObjectHeader::parse(file_data, ext_addr as usize, offset_size, length_size)?;
+    let ext = ObjectHeader::parse_in(file_data, ext_addr, offset_size, length_size)?;
     let Some(msg) = ext
         .messages
         .iter()
@@ -430,9 +477,9 @@ pub fn load_sohm_table(
         return Ok(None);
     };
     let table_msg = parse_sohm_table_message(&msg.data, offset_size)?;
-    parse_sohm_table(
+    parse_sohm_table_in(
         file_data,
-        table_msg.table_address as usize,
+        table_msg.table_address,
         table_msg.nindexes,
         offset_size,
     )
@@ -446,17 +493,27 @@ pub fn message_data_with_sohm<'a>(
     msg: &'a crate::object_header::HeaderMessage,
     offset_size: u8,
     length_size: u8,
+) -> Result<Cow<'a, [u8]>, FormatError> {
+    message_data_with_sohm_in(file_data, msg, offset_size, length_size)
+}
+
+/// [`message_data_with_sohm`] over any [`Storage`].
+pub fn message_data_with_sohm_in<'a, S: Storage + ?Sized>(
+    file_data: &S,
+    msg: &'a crate::object_header::HeaderMessage,
+    offset_size: u8,
+    length_size: u8,
 ) -> Result<Cow<'a, [u8]>, FormatError> {
     if !is_shared(msg.flags) {
         return Ok(Cow::Borrowed(&msg.data));
     }
     let shared_ref = parse_shared_ref_sized(&msg.data, offset_size, length_size)?;
     let table = if shared_ref.heap_id.is_some() {
-        load_sohm_table(file_data, offset_size, length_size)?
+        load_sohm_table_in(file_data, offset_size, length_size)?
     } else {
         None
     };
-    resolve_shared_message_with_sohm(
+    resolve_shared_message_with_sohm_in(
         file_data,
         &shared_ref,
         msg.msg_type,
@@ -495,6 +552,25 @@ pub fn resolve_sohm_message(
     target_msg_type: MessageType,
     offset_size: u8,
     length_size: u8,
+) -> Result<Vec<u8>, FormatError> {
+    resolve_sohm_message_in(
+        &file_data,
+        heap_id,
+        sohm_table,
+        target_msg_type,
+        offset_size,
+        length_size,
+    )
+}
+
+/// [`resolve_sohm_message`] over any [`Storage`].
+pub fn resolve_sohm_message_in<S: Storage + ?Sized>(
+    file_data: &S,
+    heap_id: &[u8; FHEAP_ID_LEN],
+    sohm_table: &SohmTable,
+    target_msg_type: MessageType,
+    offset_size: u8,
+    length_size: u8,
 ) -> Result<Vec<u8>, FormatError> {
     let index = find_index_for_msg_type(sohm_table, target_msg_type)
         .ok_or(FormatError::InvalidSharedMessageVersion(2))?;
@@ -503,13 +579,9 @@ pub fn resolve_sohm_message(
         return Err(FormatError::InvalidSharedMessageVersion(2));
     }
 
-    let fh_header = FractalHeapHeader::parse(
+    let fh_header =
-        file_data,
+        FractalHeapHeader::parse_in(file_data, index.heap_addr, offset_size, length_size)?;
-        index.heap_addr as usize,
+    fh_header.read_managed_object_in(file_data, heap_id, offset_size)
-        offset_size,
-        length_size,
-    )?;
-    fh_header.read_managed_object(file_data, heap_id, offset_size)
 }
 
 /// The payload of an object-header message, following the indirection if the
@@ -526,12 +598,22 @@ pub fn message_data<'a>(
     msg: &'a crate::object_header::HeaderMessage,
     offset_size: u8,
     length_size: u8,
+) -> Result<Cow<'a, [u8]>, FormatError> {
+    message_data_in(file_data, msg, offset_size, length_size)
+}
+
+/// [`message_data`] over any [`Storage`].
+pub fn message_data_in<'a, S: Storage + ?Sized>(
+    file_data: &S,
+    msg: &'a crate::object_header::HeaderMessage,
+    offset_size: u8,
+    length_size: u8,
 ) -> Result<Cow<'a, [u8]>, FormatError> {
     if !is_shared(msg.flags) {
         return Ok(Cow::Borrowed(&msg.data));
     }
     let shared_ref = parse_shared_ref_sized(&msg.data, offset_size, length_size)?;
-    resolve_shared_message(
+    resolve_shared_message_in(
         file_data,
         &shared_ref,
         msg.msg_type,
@@ -553,13 +635,30 @@ pub fn resolve_shared_message(
     target_msg_type: MessageType,
     offset_size: u8,
     length_size: u8,
+) -> Result<Vec<u8>, FormatError> {
+    resolve_shared_message_in(
+        &file_data,
+        shared_ref,
+        target_msg_type,
+        offset_size,
+        length_size,
+    )
+}
+
+/// [`resolve_shared_message`] over any [`Storage`].
+pub fn resolve_shared_message_in<S: Storage + ?Sized>(
+    file_data: &S,
+    shared_ref: &SharedMessageRef,
+    target_msg_type: MessageType,
+    offset_size: u8,
+    length_size: u8,
 ) -> Result<Vec<u8>, FormatError> {
     let table = if shared_ref.heap_id.is_some() {
-        load_sohm_table(file_data, offset_size, length_size)?
+        load_sohm_table_in(file_data, offset_size, length_size)?
     } else {
         None
     };
-    resolve_shared_message_with_sohm(
+    resolve_shared_message_with_sohm_in(
         file_data,
         shared_ref,
         target_msg_type,
@@ -577,6 +676,25 @@ pub fn resolve_shared_message_with_sohm(
     offset_size: u8,
     length_size: u8,
     sohm_table: Option<&SohmTable>,
+) -> Result<Vec<u8>, FormatError> {
+    resolve_shared_message_with_sohm_in(
+        &file_data,
+        shared_ref,
+        target_msg_type,
+        offset_size,
+        length_size,
+        sohm_table,
+    )
+}
+
+/// [`resolve_shared_message_with_sohm`] over any [`Storage`].
+pub fn resolve_shared_message_with_sohm_in<S: Storage + ?Sized>(
+    file_data: &S,
+    shared_ref: &SharedMessageRef,
+    target_msg_type: MessageType,
+    offset_size: u8,
+    length_size: u8,
+    sohm_table: Option<&SohmTable>,
 ) -> Result<Vec<u8>, FormatError> {
     // Dispatch on what the reference carries rather than on `ref_type`: v1/v2
     // references are always an object-header address whatever their type
@@ -586,8 +704,7 @@ pub fn resolve_shared_message_with_sohm(
         shared_ref.heap_id.as_ref(),
     ) {
         (Some(addr), _) => {
-            let target_header =
+            let target_header = ObjectHeader::parse_in(file_data, addr, offset_size, length_size)?;
-                ObjectHeader::parse(file_data, addr as usize, offset_size, length_size)?;
             for msg in &target_header.messages {
                 if msg.msg_type == target_msg_type && !is_shared(msg.flags) {
                     return Ok(msg.data.clone());
@@ -614,7 +731,7 @@ pub fn resolve_shared_message_with_sohm(
         }
         (None, Some(heap_id)) => {
             let table = sohm_table.ok_or(FormatError::InvalidSharedMessageVersion(2))?;
-            resolve_sohm_message(
+            resolve_sohm_message_in(
                 file_data,
                 heap_id,
                 table,
@@ -1052,4 +1169,82 @@ mod tests {
         // With 2-byte offsets: OH=2+2=4, heap=12, entry=1+4+12=17
         assert_eq!(sohm_entry_size(2), 17);
     }
+
+    /// SOHM tables and lists parse identically through a read_at-only
+    /// CountingStorage: at two offsets, with 4- and 8-byte offsets, cut at
+    /// every length and with a bad signature.
+    #[test]
+    fn storage_reads_match_slice_reads() {
+        use crate::storage::CountingStorage;
+        let idx = |t: u8, n: u16| SohmIndex {
+            index_type: t,
+            mesg_types: 0x0008,
+            min_mesg_size: 50,
+            list_max: 50,
+            btree_min: 40,
+            num_messages: n,
+            index_addr: 0x3000,
+            heap_addr: 0x4000,
+        };
+        let heap_entry = |h: u32| SohmEntry {
+            location: 0,
+            hash: h,
+            heap_id: Some([1, 2, 3, 4, 5, 6, 7, h as u8]),
+            ref_count: Some(h),
+            mesg_index: None,
+            oh_addr: None,
+        };
+        let oh_entry = SohmEntry {
+            location: 1,
+            hash: 9,
+            heap_id: None,
+            ref_count: None,
+            mesg_index: Some(3),
+            oh_addr: Some(0x7000),
+        };
+        let mut compared = 0;
+        for os in [4u8, 8] {
+            let smtb = build_smtb(&[idx(0, 2), idx(1, 7)], os);
+            let smli = build_smli(&[heap_entry(1), oh_entry.clone(), heap_entry(2)], os);
+            for (body, n) in [(smtb, 2u16), (smli, 3)] {
+                let is_table = &body[..4] == b"SMTB";
+                for at in [0usize, 0x40] {
+                    let mut full = vec![0u8; at];
+                    full.extend_from_slice(&body);
+                    let mut files = Vec::new();
+                    for cut in at..=full.len() {
+                        files.push(full[..cut].to_vec());
+                    }
+                    let mut bad = full.clone();
+                    bad[at] = b'X';
+                    files.push(bad);
+                    for f in files {
+                        let st = CountingStorage::new(f.clone());
+                        let (want, got) = if is_table {
+                            (
+                                format!("{:?}", parse_sohm_table(&f, at, n as u8, os)),
+                                format!("{:?}", parse_sohm_table_in(&st, at as u64, n as u8, os)),
+                            )
+                        } else {
+                            (
+                                format!("{:?}", parse_sohm_list(&f, at, n, os)),
+                                format!("{:?}", parse_sohm_list_in(&st, at as u64, n, os)),
+                            )
+                        };
+                        assert_eq!(got, want, "{} bytes", f.len());
+                        assert!(st.reads() <= 2);
+                        compared += 1;
+                    }
+                }
+            }
+        }
+        assert!(compared > 200);
+        // The B-tree index reads through Storage too, errors included.
+        let junk = vec![0u8; 64];
+        let st = CountingStorage::new(junk.clone());
+        assert_eq!(
+            format!("{:?}", parse_sohm_btree_entries_in(&st, 0, 8, 8)),
+            format!("{:?}", parse_sohm_btree_entries(&junk, 0, 8, 8))
+        );
+    }
 }

crates/clawhdf5-format/src/signature.rs

@@ -1,6 +1,7 @@
 //! HDF5 file signature (magic bytes) detection.
 
 use crate::error::FormatError;
+use crate::storage::{Storage, read_exact_at};
 
 /// The 8-byte HDF5 magic signature.
 pub const HDF5_SIGNATURE: [u8; 8] = [0x89, b'H', b'D', b'F', b'\r', b'\n', 0x1A, b'\n'];
@@ -39,6 +40,20 @@ pub fn find_signature(data: &[u8]) -> Result<usize, FormatError> {
     Err(FormatError::SignatureNotFound)
 }
 
+/// [`find_signature`] over any [`Storage`]: one 8-byte read per candidate
+/// offset.
+pub fn find_signature_in<S: Storage + ?Sized>(file: &S) -> Result<u64, FormatError> {
+    let len = file.len();
+    let mut offset = 0u64;
+    while offset.checked_add(8).is_some_and(|end| end <= len) {
+        if *read_exact_at(file, offset, 8)? == HDF5_SIGNATURE {
+            return Ok(offset);
+        }
+        offset = if offset == 0 { 512 } else { offset * 2 };
+    }
+    Err(FormatError::SignatureNotFound)
+}
+
 /// Split a file into its user block and its HDF5 bytes.
 ///
 /// Returns `(user_block, hdf5)`: `user_block` is everything before the
@@ -132,4 +147,26 @@ mod tests {
         data[512..520].copy_from_slice(&HDF5_SIGNATURE);
         assert_eq!(find_signature(&data), Ok(0));
     }
+
+    #[test]
+    fn find_signature_in_matches_slice_search() {
+        use crate::storage::CountingStorage;
+        for (len, at) in [
+            (0, None),
+            (7, None),
+            (8, Some(0)),
+            (600, Some(512)),
+            (5000, Some(4096)),
+            (3000, Some(2048)),
+            (3000, None),
+        ] {
+            let mut data = vec![0u8; len];
+            if let Some(at) = at {
+                data[at..at + 8].copy_from_slice(&HDF5_SIGNATURE);
+            }
+            let want = find_signature(&data).map(|o| o as u64);
+            let got = find_signature_in(&CountingStorage::new(data));
+            assert_eq!(got, want, "{len} {at:?}");
+        }
+    }
 }

crates/clawhdf5-format/src/storage.rs

@@ -0,0 +1,766 @@
+//! Where the parsers read the file from: the [`Storage`] trait.
+//!
+//! Every parser used to take the whole file as one `&[u8]`. [`Storage`] is
+//! the abstraction that replaces it (see `docs/design/range-reads.md`,
+//! option (a)): a parser asks for the bytes it needs, `[offset, offset +
+//! len)`, with 64-bit offsets, and gets them back as a [`Cow`] — borrowed
+//! when the backend holds the file in memory (a `Vec`, an mmap), owned when
+//! it had to fetch them (a range request, a block cache).
+//!
+//! `impl Storage for [u8]` serves the in-memory case with no copy, and
+//! [`Storage::as_contiguous`] lets a hot loop borrow the whole file at once
+//! when the backend has it. Modules are converted one at a time: a converted
+//! parser has an `*_in<S: Storage + ?Sized>(file: &S, ..)` core and keeps
+//! its old `&[u8]` signature as a thin wrapper, so callers do not change.
+//!
+//! The cores are generic rather than taking `&dyn Storage` so that the
+//! wrappers monomorphise for `[u8]`: the bounds check of each structure read
+//! inlines to what the slice code did, with no indirect call and no copy,
+//! which keeps local files as fast as before the migration. A `&dyn Storage`
+//! still works (`S = dyn Storage`), and a remote backend pays one indirect
+//! call per structure read.
+//!
+//! # Slice entry points
+//!
+//! A generic core is instantiated in the crate that calls it, so a
+//! downstream crate calling `parse_in::<[u8]>` gets its own copy of the
+//! parser, compiled without this crate's private helpers inlined (there is
+//! no cross-crate inlining of non-`#[inline]` functions without LTO): a
+//! metadata walk through the facade ran about 6% slower that way than
+//! through the `&[u8]` wrappers. The `*_in` entry points on the facade's hot
+//! paths (object headers, group listing and lookup, attributes) therefore
+//! check [`Storage::as_contiguous`] first and hand an in-memory file to
+//! their non-generic `&[u8]` wrapper, compiled here; both run the one core.
+//!
+//! The trait is synchronous and `no_std`: parsing is CPU work, and a remote
+//! backend bridges to its own I/O.
+
+#[cfg(not(feature = "std"))]
+use alloc::{borrow::Cow, boxed::Box, vec::Vec};
+#[cfg(feature = "std")]
+use std::{borrow::Cow, boxed::Box, vec::Vec};
+
+use core::ops::Range;
+
+use crate::error::FormatError;
+
+/// A random-access source of file bytes.
+///
+/// Offsets are relative to the start of the HDF5 data (the superblock), like
+/// every address in the file.
+pub trait Storage {
+    /// Bytes `[offset, offset + len)`.
+    ///
+    /// The result is shorter than `len` only when the range runs past the
+    /// end of the storage (and empty when `offset` is at or past the end);
+    /// a backend that cannot serve a range returns an error instead of a
+    /// short read.
+    /// It is never longer than `len`; the parsers cut a longer result to
+    /// `len` (see [`exact_len`]) rather than read bytes from outside the
+    /// range.
+    fn read_at(&self, offset: u64, len: usize) -> Result<Cow<'_, [u8]>, FormatError>;
+
+    /// Current length of the storage in bytes.
+    fn len(&self) -> u64;
+
+    /// Whether the storage holds no bytes.
+    fn is_empty(&self) -> bool {
+        self.len() == 0
+    }
+
+    /// Several reads at once, in the order given. Backends that talk to a
+    /// remote store coalesce and parallelise these; the default reads them
+    /// one by one with [`Storage::read_at`].
+    fn read_ranges(&self, ranges: &[Range<u64>]) -> Result<Vec<Cow<'_, [u8]>>, FormatError> {
+        ranges
+            .iter()
+            .map(|r| {
+                let len = usize::try_from(r.end.saturating_sub(r.start)).map_err(|_| {
+                    FormatError::Overflow("read range longer than the address space".into())
+                })?;
+                self.read_at(r.start, len)
+            })
+            .collect()
+    }
+
+    /// The whole storage as one slice, when the backend has it in memory
+    /// (a `Vec`, an mmap). Hot loops use this to keep their zero-copy path;
+    /// `None` means every byte has to go through [`Storage::read_at`].
+    fn as_contiguous(&self) -> Option<&[u8]> {
+        None
+    }
+
+    /// A hint that `[offset, offset + len)` is about to be read by the
+    /// same operation: a parser that has just learnt where the next
+    /// structures are (a node's children, a structure's body) says so
+    /// before it reads them one at a time. Nothing is read and nothing
+    /// fails. The default ignores it, as does every backend that reads
+    /// when asked; the browser's restartable reader, which fetches over
+    /// the network between attempts, fetches hinted bytes along with the
+    /// bytes an attempt actually missed, so structures a parser only
+    /// reaches after a miss arrive in the same round trip. Results never
+    /// depend on hints.
+    #[inline]
+    fn hint(&self, offset: u64, len: usize) {
+        let _ = (offset, len);
+    }
+}
+
+impl Storage for [u8] {
+    #[inline]
+    fn read_at(&self, offset: u64, len: usize) -> Result<Cow<'_, [u8]>, FormatError> {
+        let n = self.len();
+        let start = usize::try_from(offset).map_or(n, |o| o.min(n));
+        let end = start.saturating_add(len).min(n);
+        Ok(Cow::Borrowed(&self[start..end]))
+    }
+
+    #[inline]
+    fn len(&self) -> u64 {
+        <[u8]>::len(self) as u64
+    }
+
+    #[inline]
+    fn as_contiguous(&self) -> Option<&[u8]> {
+        Some(self)
+    }
+}
+
+impl Storage for Vec<u8> {
+    #[inline]
+    fn read_at(&self, offset: u64, len: usize) -> Result<Cow<'_, [u8]>, FormatError> {
+        self.as_slice().read_at(offset, len)
+    }
+
+    #[inline]
+    fn len(&self) -> u64 {
+        Vec::len(self) as u64
+    }
+
+    #[inline]
+    fn as_contiguous(&self) -> Option<&[u8]> {
+        Some(self.as_slice())
+    }
+}
+
+impl<T: Storage + ?Sized> Storage for &T {
+    #[inline]
+    fn read_at(&self, offset: u64, len: usize) -> Result<Cow<'_, [u8]>, FormatError> {
+        (**self).read_at(offset, len)
+    }
+
+    #[inline]
+    fn len(&self) -> u64 {
+        (**self).len()
+    }
+
+    #[inline]
+    fn read_ranges(&self, ranges: &[Range<u64>]) -> Result<Vec<Cow<'_, [u8]>>, FormatError> {
+        (**self).read_ranges(ranges)
+    }
+
+    #[inline]
+    fn as_contiguous(&self) -> Option<&[u8]> {
+        (**self).as_contiguous()
+    }
+
+    #[inline]
+    fn hint(&self, offset: u64, len: usize) {
+        (**self).hint(offset, len)
+    }
+}
+
+impl<T: Storage + ?Sized> Storage for Box<T> {
+    #[inline]
+    fn read_at(&self, offset: u64, len: usize) -> Result<Cow<'_, [u8]>, FormatError> {
+        (**self).read_at(offset, len)
+    }
+
+    #[inline]
+    fn len(&self) -> u64 {
+        (**self).len()
+    }
+
+    #[inline]
+    fn read_ranges(&self, ranges: &[Range<u64>]) -> Result<Vec<Cow<'_, [u8]>>, FormatError> {
+        (**self).read_ranges(ranges)
+    }
+
+    #[inline]
+    fn as_contiguous(&self) -> Option<&[u8]> {
+        (**self).as_contiguous()
+    }
+
+    #[inline]
+    fn hint(&self, offset: u64, len: usize) {
+        (**self).hint(offset, len)
+    }
+}
+
+#[cfg(feature = "std")]
+impl<T: Storage + ?Sized> Storage for std::sync::Arc<T> {
+    #[inline]
+    fn read_at(&self, offset: u64, len: usize) -> Result<Cow<'_, [u8]>, FormatError> {
+        (**self).read_at(offset, len)
+    }
+
+    #[inline]
+    fn len(&self) -> u64 {
+        (**self).len()
+    }
+
+    #[inline]
+    fn read_ranges(&self, ranges: &[Range<u64>]) -> Result<Vec<Cow<'_, [u8]>>, FormatError> {
+        (**self).read_ranges(ranges)
+    }
+
+    #[inline]
+    fn as_contiguous(&self) -> Option<&[u8]> {
+        (**self).as_contiguous()
+    }
+
+    #[inline]
+    fn hint(&self, offset: u64, len: usize) {
+        (**self).hint(offset, len)
+    }
+}
+
+/// `storage.len()` as the `usize` the parsers' end-of-file errors report
+/// (saturating on targets where the file is larger than the address space).
+#[inline]
+pub(crate) fn len_usize<S: Storage + ?Sized>(file: &S) -> usize {
+    usize::try_from(file.len()).unwrap_or(usize::MAX)
+}
+
+/// Read `len` bytes at `offset` and drop them, ignoring any error.
+///
+/// For a traversal that has failed on one sibling (a B-tree child, a
+/// symbol table node, a heap object) and would stop there: it first
+/// touches the siblings it did not get to, so a storage that records what
+/// it lacks — the browser's restartable reader, which fetches over the
+/// network between attempts — learns about all of them in one attempt
+/// instead of one per attempt. Results and errors are unchanged (the first
+/// error is still the one returned); an in-memory read is free.
+pub fn touch<S: Storage + ?Sized>(file: &S, offset: u64, len: usize) {
+    let _ = file.read_at(offset, len);
+}
+
+/// Bytes `[offset, offset + len)`, all of them.
+///
+/// A range that runs past the end of the storage is
+/// [`FormatError::UnexpectedEof`] with `expected = offset + len` and
+/// `available = storage length` — the error the `&[u8]` parsers give for
+/// the same bounds check (`offset + len > file_data.len()`).
+#[inline]
+pub fn read_exact_at<S: Storage + ?Sized>(
+    file: &S,
+    offset: u64,
+    len: usize,
+) -> Result<Cow<'_, [u8]>, FormatError> {
+    let eof = || FormatError::UnexpectedEof {
+        expected: usize::try_from(offset)
+            .unwrap_or(usize::MAX)
+            .saturating_add(len),
+        available: len_usize(file),
+    };
+    // In-memory fast path: plain slicing (for `S = [u8]` this inlines to
+    // the slice code's bounds check).
+    if let Some(all) = file.as_contiguous() {
+        return usize::try_from(offset)
+            .ok()
+            .and_then(|start| all.get(start..start.checked_add(len)?))
+            .map(Cow::Borrowed)
+            .ok_or_else(eof);
+    }
+    match offset.checked_add(len as u64) {
+        Some(end) if end <= file.len() => {}
+        _ => return Err(eof()),
+    }
+    // A short read (the storage shrank, or the backend served less inside
+    // the file) is an error: never parse a partial structure.
+    exact_len(file.read_at(offset, len)?, len)
+}
+
+/// `bytes`, the result of asking a [`Storage`] for `len` bytes, as exactly
+/// `len` bytes: a longer result (a backend that broke
+/// [`Storage::read_at`]'s contract) is cut to `len`, so bytes from outside
+/// the range asked for are never parsed or returned; a shorter one is an
+/// error (the storage shrank, or the backend failed), never a partial
+/// structure.
+#[inline]
+pub fn exact_len(bytes: Cow<'_, [u8]>, len: usize) -> Result<Cow<'_, [u8]>, FormatError> {
+    match bytes.len().cmp(&len) {
+        core::cmp::Ordering::Equal => Ok(bytes),
+        core::cmp::Ordering::Less => Err(short_read()),
+        core::cmp::Ordering::Greater => Ok(match bytes {
+            Cow::Borrowed(b) => Cow::Borrowed(&b[..len]),
+            Cow::Owned(mut v) => {
+                v.truncate(len);
+                Cow::Owned(v)
+            }
+        }),
+    }
+}
+
+#[cold]
+#[inline(never)]
+fn short_read() -> FormatError {
+    FormatError::Storage(
+        "short read inside the file (the storage shrank or the backend failed)".into(),
+    )
+}
+
+/// Largest paged data block (fixed or extensible array) read in one piece.
+/// A bigger one is read as its prefix and then page by page, only the pages
+/// in use, so a block whose size fields claim more than the file holds
+/// costs no more than the pages it really has.
+pub(crate) const PAGED_BLOCK_ONE_READ_MAX: usize = 1 << 20;
+
+/// A window of the file: up to `max` bytes read at `base`, fewer only at
+/// the end of the file. Its [`Window::ensure`] reports a bounds failure
+/// exactly as the whole-file check `ensure_len(file_data, base + rel, n)`
+/// did — with the absolute position and the file's length — as long as
+/// every position checked lies within the `max` bytes the window was asked
+/// for: then a position past the window is past the end of the file.
+pub(crate) struct Window<'a> {
+    /// The bytes, from `base` on.
+    pub bytes: Cow<'a, [u8]>,
+    base: usize,
+    file_len: usize,
+}
+
+impl<'a> Window<'a> {
+    /// Read up to `max` bytes at `base`.
+    pub fn read<S: Storage + ?Sized>(
+        file: &'a S,
+        base: u64,
+        max: usize,
+    ) -> Result<Self, FormatError> {
+        Ok(Window {
+            bytes: read_upto(file, base, max)?,
+            base: usize::try_from(base).unwrap_or(usize::MAX),
+            file_len: len_usize(file),
+        })
+    }
+
+    /// A whole in-memory file as one window (base 0).
+    #[cfg(test)]
+    pub fn whole(bytes: &'a [u8]) -> Self {
+        Window {
+            bytes: Cow::Borrowed(bytes),
+            base: 0,
+            file_len: bytes.len(),
+        }
+    }
+
+    /// [`Window::ensure`] for a window at `base` that has not been read:
+    /// whether `[rel, rel + needed)` lies in the file, with the same error.
+    /// Lets a parser whose first step is to check a structure's whole extent
+    /// (a checksum at its end) fail before reading a structure that a
+    /// hostile size field has stretched past the end of the file.
+    pub fn check_extent<S: Storage + ?Sized>(
+        file: &S,
+        base: u64,
+        rel: usize,
+        needed: usize,
+    ) -> Result<(), FormatError> {
+        let base = usize::try_from(base).unwrap_or(usize::MAX);
+        let file_len = len_usize(file);
+        match base.checked_add(rel).and_then(|p| p.checked_add(needed)) {
+            Some(end) if end <= file_len => Ok(()),
+            _ => Err(FormatError::UnexpectedEof {
+                expected: base.saturating_add(rel).saturating_add(needed),
+                available: file_len,
+            }),
+        }
+    }
+
+    /// Check that `[rel, rel + needed)` (relative to `base`) is in the file.
+    #[inline]
+    pub fn ensure(&self, rel: usize, needed: usize) -> Result<(), FormatError> {
+        match rel.checked_add(needed) {
+            Some(end) if end <= self.bytes.len() => Ok(()),
+            _ => Err(FormatError::UnexpectedEof {
+                expected: self.base.saturating_add(rel).saturating_add(needed),
+                available: self.file_len,
+            }),
+        }
+    }
+}
+
+/// Up to `max` bytes from `offset` on: fewer only at the end of the
+/// storage. For structures whose size is only known once their prefix has
+/// been parsed and whose parsers bound-check what they are given.
+#[inline]
+pub fn read_upto<S: Storage + ?Sized>(
+    file: &S,
+    offset: u64,
+    max: usize,
+) -> Result<Cow<'_, [u8]>, FormatError> {
+    if let Some(all) = file.as_contiguous() {
+        let start = usize::try_from(offset).map_or(all.len(), |o| o.min(all.len()));
+        let end = start.saturating_add(max).min(all.len());
+        return Ok(Cow::Borrowed(&all[start..end]));
+    }
+    let avail = file.len().saturating_sub(offset);
+    let len = usize::try_from(avail).map_or(max, |a| a.min(max));
+    exact_len(file.read_at(offset, len)?, len)
+}
+
+/// Most stored bytes fetched by one [`Storage::read_ranges`] call when a
+/// read gathers many extents (a chunked dataset's chunks, a selection's
+/// runs): a larger read is fetched and decoded batch by batch, so a backend
+/// without the file in memory never holds more than this much undecoded
+/// data per read (or one extent, when a single one is larger — and every
+/// chunk's extent is bounded by what the chunk can need, see
+/// [`crate::filters::stored_chunk_limit`]).
+pub const RAW_BATCH_BYTES: usize = 64 << 20;
+
+/// One extent of a raw-data read: `len` bytes stored at `addr`, whose
+/// bounds are checked against the file, of which the first `fetch` bytes
+/// are read (`None`: only checked, not read — its bytes are not needed).
+///
+/// `fetch` below `len` bounds what a crafted size field can make a read
+/// fetch: a chunk never needs more of its stored bytes than its decoded
+/// size allows, however large its index entry says it is.
+#[derive(Debug, Clone, Copy)]
+pub(crate) struct ExtentReq {
+    pub addr: u64,
+    pub len: usize,
+    pub fetch: Option<usize>,
+}
+
+impl ExtentReq {
+    /// How many bytes are read for this extent.
+    #[inline]
+    fn fetch_len(&self) -> usize {
+        self.fetch.map_or(0, |f| f.min(self.len))
+    }
+}
+
+/// One extent's bytes on their own (see [`ExtentReq`]): the whole extent's
+/// bounds checked as [`read_exact_at`] checks them, and its first
+/// `req.fetch` bytes read (none when `fetch` is `None`).
+pub(crate) fn read_extent<'a, S: Storage + ?Sized>(
+    file: &'a S,
+    req: &ExtentReq,
+) -> Result<Cow<'a, [u8]>, FormatError> {
+    let start = usize::try_from(req.addr).unwrap_or(usize::MAX);
+    match start.checked_add(req.len) {
+        Some(end) if end <= len_usize(file) => read_exact_at(file, req.addr, req.fetch_len()),
+        _ => Err(FormatError::UnexpectedEof {
+            expected: start.saturating_add(req.len),
+            available: len_usize(file),
+        }),
+    }
+}
+
+/// The stored bytes of one batch of extents (chunks, contiguous runs),
+/// fetched together: [`Storage::read_ranges`] is called once for the batch,
+/// so a remote backend can coalesce and parallelise the requests. See
+/// [`for_each_extent_batch`], which is how every raw-data read gets them.
+///
+/// With the whole file in memory nothing is fetched: [`Self::get`] slices
+/// it, as the slice readers did. Either way an extent that does not lie in
+/// the file is the error the slice readers gave for it
+/// ([`FormatError::UnexpectedEof`] with its end and the file length, or
+/// [`FormatError::Overflow`] for an address past this platform's `usize`),
+/// reported when that extent is asked for — so a read reports the first
+/// failing extent in its own order, whatever fails after it.
+pub(crate) enum ExtentBytes<'a> {
+    /// The whole file.
+    Contiguous(&'a [u8]),
+    /// Each extent's bytes, or its bounds error; the first is extent
+    /// `base` of the read.
+    Fetched {
+        base: usize,
+        extents: Vec<Extent<'a>>,
+    },
+}
+
+/// One extent of [`ExtentBytes::Fetched`].
+pub(crate) enum Extent<'a> {
+    /// Its bytes.
+    Bytes(Cow<'a, [u8]>),
+    /// In the file, but not fetched (the caller did not want its bytes).
+    NotFetched,
+    /// The error reading it gives.
+    Err(FormatError),
+}
+
+impl<'a> ExtentBytes<'a> {
+    /// Fetch `reqs`, extents `base..base + reqs.len()` of the read: the
+    /// bytes of those wanted, and the bounds check of all of them.
+    pub(crate) fn fetch<S: Storage + ?Sized>(
+        file: &'a S,
+        reqs: &[ExtentReq],
+        base: usize,
+    ) -> Result<Self, FormatError> {
+        if let Some(all) = file.as_contiguous() {
+            return Ok(ExtentBytes::Contiguous(all));
+        }
+        let file_len = len_usize(file);
+        let mut ranges = Vec::new();
+        let mut out = Vec::with_capacity(reqs.len());
+        // Positions in `out` of the extents being read, in `ranges` order.
+        let mut slots = Vec::new();
+        for req in reqs {
+            let checked = crate::addr::to_usize(req.addr).and_then(|start| {
+                match start.checked_add(req.len) {
+                    Some(end) if end <= file_len => Ok(()),
+                    _ => Err(FormatError::UnexpectedEof {
+                        expected: start.saturating_add(req.len),
+                        available: file_len,
+                    }),
+                }
+            });
+            match checked {
+                Ok(()) if req.fetch.is_some() => {
+                    slots.push(out.len());
+                    ranges.push(req.addr..req.addr + req.fetch_len() as u64);
+                    out.push(Extent::NotFetched);
+                }
+                Ok(()) => out.push(Extent::NotFetched),
+                Err(e) => out.push(Extent::Err(e)),
+            }
+        }
+        if !ranges.is_empty() {
+            let got = file.read_ranges(&ranges)?;
+            if got.len() != ranges.len() {
+                return Err(FormatError::Storage(
+                    "read_ranges returned the wrong number of ranges".into(),
+                ));
+            }
+            for ((slot, bytes), r) in slots.into_iter().zip(got).zip(&ranges) {
+                let len = crate::addr::saturating_usize(r.end - r.start);
+                out[slot] = Extent::Bytes(exact_len(bytes, len)?);
+            }
+        }
+        Ok(ExtentBytes::Fetched { base, extents: out })
+    }
+
+    /// Whether extent `i` of the read (`req`) lies in the file: its bounds
+    /// error if not.
+    pub(crate) fn check(&self, i: usize, req: &ExtentReq) -> Result<(), FormatError> {
+        match self {
+            ExtentBytes::Contiguous(_) => self.get(i, req).map(|_| ()),
+            ExtentBytes::Fetched { base, extents } => {
+                match i.checked_sub(*base).and_then(|j| extents.get(j)) {
+                    Some(Extent::Err(e)) => Err(e.clone()),
+                    Some(_) => Ok(()),
+                    None => Err(not_fetched()),
+                }
+            }
+        }
+    }
+
+    /// Extent `i` of the read (`req`): its first `req.fetch` bytes, the
+    /// same whether the file is in memory or not.
+    pub(crate) fn get(&self, i: usize, req: &ExtentReq) -> Result<&[u8], FormatError> {
+        match self {
+            ExtentBytes::Contiguous(all) => {
+                let start = crate::addr::to_usize(req.addr)?;
+                start
+                    .checked_add(req.len)
+                    .and_then(|end| all.get(start..end))
+                    .map(|b| &b[..req.fetch_len()])
+                    .ok_or(FormatError::UnexpectedEof {
+                        expected: start.saturating_add(req.len),
+                        available: <[u8]>::len(all),
+                    })
+            }
+            ExtentBytes::Fetched { base, extents } => {
+                match i.checked_sub(*base).and_then(|j| extents.get(j)) {
+                    Some(Extent::Bytes(b)) => Ok(b),
+                    Some(Extent::Err(e)) => Err(e.clone()),
+                    _ => Err(not_fetched()),
+                }
+            }
+        }
+    }
+}
+
+#[cold]
+fn not_fetched() -> FormatError {
+    FormatError::Storage("an extent that was not fetched was asked for".into())
+}
+
+/// The one way raw-data reads fetch stored bytes: `reqs` are split into
+/// consecutive batches of at most [`RAW_BATCH_BYTES`] of fetched bytes (at
+/// least one extent each — and no extent fetches more than its
+/// [`ExtentReq::fetch`]), and for each batch in turn its bytes are fetched
+/// with one [`Storage::read_ranges`] call and `f(batch, &bytes)` is called,
+/// with `bytes` indexed by the extent's position in `reqs`. A batch's bytes
+/// are dropped before the next batch is fetched, and an error from `f`
+/// stops the read before anything more is fetched.
+///
+/// With the whole file in memory there is nothing to fetch: one call, over
+/// all of `reqs`, that slices the file.
+pub(crate) fn for_each_extent_batch<'a, S: Storage + ?Sized>(
+    file: &'a S,
+    reqs: &[ExtentReq],
+    mut f: impl FnMut(Range<usize>, &ExtentBytes<'a>) -> Result<(), FormatError>,
+) -> Result<(), FormatError> {
+    let contiguous = file.as_contiguous().is_some();
+    for batch in raw_batches(reqs.len(), contiguous, |i| reqs[i].fetch_len()) {
+        let bytes = ExtentBytes::fetch(file, &reqs[batch.clone()], batch.start)?;
+        f(batch, &bytes)?;
+    }
+    Ok(())
+}
+
+/// Split `n` extents, whose sizes `size(i)` gives, into consecutive batches
+/// of at most [`RAW_BATCH_BYTES`] (at least one extent each): the ranges of
+/// `0..n` to fetch together. With the whole file in memory (`contiguous`)
+/// there is nothing to fetch, and one batch.
+pub(crate) fn raw_batches(
+    n: usize,
+    contiguous: bool,
+    size: impl Fn(usize) -> usize,
+) -> Vec<Range<usize>> {
+    if contiguous || n == 0 {
+        return core::iter::once(0..n).collect();
+    }
+    let mut out = Vec::new();
+    let (mut start, mut bytes) = (0, 0usize);
+    for i in 0..n {
+        let s = size(i);
+        if i > start && bytes.saturating_add(s) > RAW_BATCH_BYTES {
+            out.push(start..i);
+            start = i;
+            bytes = 0;
+        }
+        bytes = bytes.saturating_add(s);
+    }
+    out.push(start..n);
+    out
+}
+
+/// Borrow the whole file for a code path that has not been converted to
+/// [`Storage`] yet. On a backend without a contiguous view this is the
+/// clean [`FormatError::ContiguousStorageRequired`] error, never a guess.
+#[inline]
+pub fn require_contiguous<'a, S: Storage + ?Sized>(
+    file: &'a S,
+    what: &'static str,
+) -> Result<&'a [u8], FormatError> {
+    file.as_contiguous()
+        .ok_or(FormatError::ContiguousStorageRequired(what))
+}
+
+/// A [`Storage`] over an in-memory buffer that serves every byte through
+/// [`Storage::read_at`] (its [`Storage::as_contiguous`] is `None`, so no
+/// parser can take the whole-slice shortcut), copies what it serves (as a
+/// remote backend would), and counts the reads and bytes.
+///
+/// It is the equivalence harness of the range-read migration: parsing a
+/// file through it must give exactly what parsing the `&[u8]` gives, and
+/// the counters are the request counts a cacheless range reader would make.
+#[derive(Debug)]
+pub struct CountingStorage {
+    data: Vec<u8>,
+    reads: portable_atomic::AtomicU64,
+    bytes: portable_atomic::AtomicU64,
+}
+
+impl CountingStorage {
+    /// Serve `data` (the file from the superblock on).
+    pub fn new(data: Vec<u8>) -> Self {
+        CountingStorage {
+            data,
+            reads: portable_atomic::AtomicU64::new(0),
+            bytes: portable_atomic::AtomicU64::new(0),
+        }
+    }
+
+    /// Number of `read_at` calls served so far.
+    pub fn reads(&self) -> u64 {
+        self.reads.load(portable_atomic::Ordering::Relaxed)
+    }
+
+    /// Number of bytes served so far.
+    pub fn bytes_read(&self) -> u64 {
+        self.bytes.load(portable_atomic::Ordering::Relaxed)
+    }
+
+    /// Reset both counters.
+    pub fn reset(&self) {
+        self.reads.store(0, portable_atomic::Ordering::Relaxed);
+        self.bytes.store(0, portable_atomic::Ordering::Relaxed);
+    }
+}
+
+impl Storage for CountingStorage {
+    fn read_at(&self, offset: u64, len: usize) -> Result<Cow<'_, [u8]>, FormatError> {
+        let got = self.data.as_slice().read_at(offset, len)?;
+        self.reads.fetch_add(1, portable_atomic::Ordering::Relaxed);
+        self.bytes
+            .fetch_add(got.len() as u64, portable_atomic::Ordering::Relaxed);
+        Ok(Cow::Owned(got.into_owned()))
+    }
+
+    fn len(&self) -> u64 {
+        self.data.len() as u64
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn slice_reads_are_borrowed_and_clamped() {
+        let data: Vec<u8> = (0u8..10).collect();
+        let s: &[u8] = &data;
+        let dynamic: &dyn Storage = &s;
+        assert_eq!(dynamic.len(), 10);
+        let r = dynamic.read_at(2, 3).unwrap();
+        assert!(matches!(r, Cow::Borrowed(_)));
+        assert_eq!(&*r, &[2, 3, 4]);
+        assert_eq!(&*dynamic.read_at(8, 5).unwrap(), &[8, 9]);
+        assert!(dynamic.read_at(10, 5).unwrap().is_empty());
+        assert!(dynamic.read_at(u64::MAX, 5).unwrap().is_empty());
+        assert_eq!(dynamic.as_contiguous(), Some(&data[..]));
+        let v: &dyn Storage = &data;
+        assert_eq!(v.as_contiguous(), Some(&data[..]));
+    }
+
+    #[test]
+    fn read_exact_matches_slice_bounds_errors() {
+        let data = [0u8; 10];
+        let s: &[u8] = &data;
+        assert_eq!(&*read_exact_at(&s, 4, 6).unwrap(), &[0; 6]);
+        assert_eq!(
+            read_exact_at(&s, 4, 7).unwrap_err(),
+            FormatError::UnexpectedEof {
+                expected: 11,
+                available: 10
+            }
+        );
+        assert!(read_exact_at(&s, u64::MAX, 1).is_err());
+        assert_eq!(read_upto(&s, 7, 100).unwrap().len(), 3);
+        assert_eq!(read_upto(&s, 70, 100).unwrap().len(), 0);
+    }
+
+    #[test]
+    fn counting_storage_counts_and_hides_the_slice() {
+        let c = CountingStorage::new((0u8..10).collect());
+        assert!(c.as_contiguous().is_none());
+        let r = c.read_at(3, 4).unwrap();
+        assert!(matches!(r, Cow::Owned(_)));
+        assert_eq!(&*r, &[3, 4, 5, 6]);
+        c.read_at(8, 4).unwrap();
+        assert_eq!((c.reads(), c.bytes_read()), (2, 6));
+        c.reset();
+        assert_eq!((c.reads(), c.bytes_read()), (0, 0));
+    }
+
+    #[test]
+    fn read_ranges_default_loops() {
+        let data: Vec<u8> = (0u8..10).collect();
+        let s: &[u8] = &data;
+        let got = s.read_ranges(&[1..3, 5..9]).unwrap();
+        assert_eq!(&*got[0], &[1, 2]);
+        assert_eq!(&*got[1], &[5, 6, 7, 8]);
+    }
+}

crates/clawhdf5-format/src/superblock.rs

@@ -7,6 +7,11 @@ use byteorder::{ByteOrder, LittleEndian};
 
 use crate::error::FormatError;
 use crate::signature::HDF5_SIGNATURE;
+use crate::storage::{Storage, read_upto};
+
+/// Bytes read to parse a superblock: more than the largest one (version 1
+/// with 8-byte offsets and lengths, 100 bytes).
+const SUPERBLOCK_READ_LEN: usize = 128;
 
 /// Parsed HDF5 superblock (all versions).
 #[derive(Debug, Clone, PartialEq, Eq)]
@@ -111,22 +116,27 @@ impl Superblock {
     /// [`FormatError::TruncatedFile`]. Bytes past that address are not part
     /// of the file: libhdf5 fails any read of them ("addr overflow" /
     /// "address plus size exceeds file eoa"), so a reader should parse only
-    /// the data up to the returned end. As libhdf5 does for a SWMR reader,
+    /// the data up to the returned end.
-    /// the check is skipped for a version-3 superblock whose writer is still
+    ///
-    /// writing it in SWMR mode (it extends the file as it goes); the data
+    /// A version-3 superblock with the SWMR-write flag set belongs to a file
-    /// then ends at the end of the file.
+    /// a SWMR writer has open (or had, and did not close). That writer does
+    /// not keep the recorded end of file up to date — a copy taken mid-write
+    /// can record an end of a few hundred bytes in a file of tens of
+    /// kilobytes — and libhdf5's SWMR reader skips its end-of-allocation
+    /// check for every read (`H5FD_read`). For such a superblock the data
+    /// ends at the end of the file, whatever end it records.
     ///
     /// When the superblock's recorded base address differs from where the
     /// superblock actually is (a user block added or removed after the file
     /// was written), libhdf5 moves the recorded end of file by the same
     /// amount, and so does this.
     pub fn data_end(&self, user_block: u64, file_len: u64) -> Result<u64, FormatError> {
-        let eof =
-            i128::from(self.eof_address) - i128::from(self.base_address) + i128::from(user_block);
-        if eof < 0 || eof > i128::from(file_len) {
         if self.version >= 3 && self.is_swmr_write() {
             return Ok(file_len.saturating_sub(user_block));
         }
+        let eof =
+            i128::from(self.eof_address) - i128::from(self.base_address) + i128::from(user_block);
+        if eof < 0 || eof > i128::from(file_len) {
             return Err(FormatError::TruncatedFile {
                 stored_eof: u64::try_from(eof).unwrap_or(self.eof_address),
                 actual_len: file_len,
@@ -161,7 +171,16 @@ impl Superblock {
         file_data: &[u8],
         signature_offset: usize,
     ) -> Result<u64, FormatError> {
-        let refreshed = Superblock::parse(file_data, signature_offset)?;
+        self.refresh_eof_in(file_data, signature_offset as u64)
+    }
+
+    /// [`Self::refresh_eof`] over any [`Storage`].
+    pub fn refresh_eof_in<S: Storage + ?Sized>(
+        &mut self,
+        file: &S,
+        signature_offset: u64,
+    ) -> Result<u64, FormatError> {
+        let refreshed = Superblock::parse_in(file, signature_offset)?;
         self.eof_address = refreshed.eof_address;
         self.consistency_flags = refreshed.consistency_flags;
         Ok(self.eof_address)
@@ -219,15 +238,23 @@ impl Superblock {
     /// [`FormatError::UserBlockNotStripped`] because the addresses in the
     /// returned superblock would otherwise be applied to the wrong bytes.
     pub fn parse(data: &[u8], signature_offset: usize) -> Result<Superblock, FormatError> {
-        if signature_offset != 0 {
+        Self::parse_in(data, signature_offset as u64)
-            return Err(FormatError::UserBlockNotStripped(signature_offset as u64));
     }
-        let d = data
+
-            .get(signature_offset..)
+    /// [`Self::parse`] over any [`Storage`]: one read of the first
-            .ok_or(FormatError::UnexpectedEof {
+    /// [`SUPERBLOCK_READ_LEN`] bytes (fewer when the file is shorter, which
-                expected: signature_offset + 1,
+    /// is then refused with the same end-of-file errors as a short slice).
-                available: data.len(),
+    pub fn parse_in<S: Storage + ?Sized>(
-            })?;
+        file: &S,
+        signature_offset: u64,
+    ) -> Result<Superblock, FormatError> {
+        if signature_offset != 0 {
+            return Err(FormatError::UserBlockNotStripped(signature_offset));
+        }
+        // Every bounds check below needs at most 100 bytes, so on a longer
+        // file none of them can fail and the window's length does not show.
+        let window = read_upto(file, 0, SUPERBLOCK_READ_LEN)?;
+        let d: &[u8] = &window;
         ensure_len(d, 9)?; // signature(8) + version(1)
 
         // Verify signature
@@ -595,6 +622,13 @@ mod tests {
         let mut swmr = Superblock::parse(&build_v2_bytes(8, 3), 0).unwrap();
         swmr.consistency_flags = swmr_flags::WRITE_ACCESS | swmr_flags::SWMR_WRITE;
         assert_eq!(swmr.data_end(0, 1000), Ok(1000));
+        // ... nor bounded by its recorded end, which the writer does not
+        // keep up to date (2048 here).
+        assert_eq!(swmr.data_end(0, 17_857), Ok(17_857));
+        assert_eq!(swmr.data_end(512, 17_857), Ok(17_345));
+        // Without the SWMR-write flag the recorded end bounds the data.
+        swmr.consistency_flags = swmr_flags::WRITE_ACCESS;
+        assert_eq!(swmr.data_end(0, 17_857), Ok(2048));
     }
 
     #[test]
@@ -894,4 +928,34 @@ mod tests {
         assert_eq!(parsed.version, 3);
         assert_eq!(parsed.page_size, None);
     }
+
+    /// Through a storage that serves only `read_at`, every version parses
+    /// to the same superblock, and every truncation to the same error, as
+    /// from a slice — in one read.
+    #[test]
+    fn parse_in_matches_slice_parse() {
+        use crate::storage::CountingStorage;
+        let mut files = vec![
+            build_v0_bytes(8),
+            build_v0_bytes(4),
+            build_v1_bytes(8),
+            build_v1_bytes(4),
+            build_v2_bytes(8, 2),
+            build_v2_bytes(4, 3),
+        ];
+        for f in files.clone() {
+            let mut long = f.clone();
+            long.resize(4096, 0xAB);
+            files.push(long);
+            for cut in [0, 5, 9, 13, 20, 30, f.len() - 1] {
+                files.push(f[..cut.min(f.len())].to_vec());
+            }
+        }
+        for f in files {
+            let want = Superblock::parse(&f, 0);
+            let storage = CountingStorage::new(f.clone());
+            assert_eq!(Superblock::parse_in(&storage, 0), want, "{} bytes", f.len());
+            assert_eq!(storage.reads(), 1);
+        }
+    }
 }

crates/clawhdf5-format/src/superblock_ext.rs

@@ -0,0 +1,922 @@
+//! The superblock extension of a version 2 or 3 superblock, and the
+//! metadata cache image it can point to.
+//!
+//! libhdf5 reads the extension when it opens a file (`H5F__super_read`) and
+//! decodes the messages that configure the file: v1 B-tree "K" values, File
+//! Space Info, and the Metadata Cache Image. A message that does not decode
+//! makes the file fail to open, so [`read_superblock_extension`] decodes and
+//! checks them the way libhdf5 does.
+//!
+//! A metadata cache image (written with `H5Pset_mdc_image_config`) is a
+//! block holding serialized metadata cache entries — object headers, B-tree
+//! nodes, heaps — each with its file address. libhdf5 loads it into its
+//! cache before it reads any other metadata (`H5C__load_cache_image`,
+//! `H5C__reconstruct_cache_contents`), and the entries take the place of
+//! the file's bytes at their addresses: the file itself may hold stale or
+//! no metadata there (in `h5clear_mdc_image.h5` the root group's header is
+//! only in the image). [`CacheImage::apply`] does the same with bytes: it
+//! writes every entry at its address, so every parser reads what libhdf5
+//! reads. It writes into whatever the opener gives it — a private
+//! copy-on-write mapping of the file, or a buffer the opener owns — so the
+//! file is never copied whole.
+
+#[cfg(not(feature = "std"))]
+use alloc::{borrow::Cow, collections::BTreeSet, vec::Vec};
+#[cfg(feature = "std")]
+use std::{borrow::Cow, collections::BTreeSet};
+
+use crate::error::FormatError;
+use crate::message_type::MessageType;
+use crate::object_header::ObjectHeader;
+use crate::storage::{Storage, read_exact_at};
+use crate::superblock::Superblock;
+
+/// Message type of the File Space Info message.
+const MSG_FSINFO: u16 = 0x0017;
+/// Message type of the Metadata Cache Image message.
+const MSG_MDCI: u16 = 0x0018;
+/// Header message flag: the library did not know the message when it wrote
+/// it back (`H5O_MSG_FLAG_WAS_UNKNOWN`); libhdf5 then ignores its contents.
+const MSG_FLAG_WAS_UNKNOWN: u8 = 0x20;
+
+/// `H5F_FILE_SPACE_PAGE_SIZE_MIN` / `_MAX`.
+const PAGE_SIZE_MIN: u64 = 512;
+const PAGE_SIZE_MAX: u64 = 1024 * 1024 * 1024;
+/// libhdf5's default file space page size, used for a version 0 message.
+const PAGE_SIZE_DEFAULT: u64 = 4096;
+/// Free-space managers whose addresses a persisting version 1 File Space
+/// Info message lists (`H5F_MEM_PAGE_SUPER` .. `H5F_MEM_PAGE_NTYPES`), and
+/// a version 0 one (`H5FD_MEM_SUPER` .. `H5FD_MEM_NTYPES`).
+const FSM_ADDRS_V1: usize = 12;
+const FSM_ADDRS_V0: usize = 6;
+
+/// Metadata cache image block limits (`H5Cimage.c`, `H5ACprivate.h`).
+const MDCI_SIGNATURE: &[u8; 4] = b"MDCI";
+const MDCI_HAVE_RESIZE_STATUS: u8 = 0x01;
+const MDCI_ENTRY_IS_FD_PARENT: u8 = 0x04;
+const MDCI_ENTRY_IS_FD_CHILD: u8 = 0x08;
+/// `H5AC_NTYPES`: entry type ids are below this.
+const MDCI_NTYPES: u8 = 30;
+/// `H5C_RING_NTYPES`.
+const MDCI_RING_NTYPES: u8 = 6;
+/// `H5AC__CACHE_IMAGE__ENTRY_AGEOUT__MAX`.
+const MDCI_AGE_MAX: u8 = 100;
+
+/// A decoded File Space Info message (0x0017), mapped to version 1 as
+/// libhdf5 maps a version 0 one.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct FileSpaceInfo {
+    /// Message version as stored (0 or 1).
+    pub version: u8,
+    /// File space strategy (`H5F_fspace_strategy_t`).
+    pub strategy: u8,
+    /// Whether free space is persisted.
+    pub persist: bool,
+    /// Free-space section threshold.
+    pub threshold: u64,
+    /// File space page size.
+    pub page_size: u64,
+}
+
+/// Where a metadata cache image block is (Metadata Cache Image message,
+/// 0x0018).
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct CacheImageLocation {
+    /// Address of the image block.
+    pub address: u64,
+    /// Length of the image block in bytes.
+    pub length: u64,
+}
+
+/// The messages of a superblock extension that libhdf5 decodes at open.
+#[derive(Debug, Clone, Default, PartialEq, Eq)]
+pub struct SuperblockExtension {
+    /// v1 B-tree "K" values (chunk index, symbol table node, symbol table
+    /// leaf), when the extension overrides the defaults.
+    pub btree_k: Option<(u16, u16, u16)>,
+    /// The File Space Info message.
+    pub file_space_info: Option<FileSpaceInfo>,
+    /// The metadata cache image, when the file has one.
+    pub cache_image: Option<CacheImageLocation>,
+}
+
+fn ext_err(why: &'static str) -> FormatError {
+    FormatError::InvalidSuperblockExtension(why)
+}
+
+const RAN_OFF: &str = "ran off end of input buffer while decoding";
+
+/// A little-endian cursor over one message or block, failing with
+/// `overrun` when it runs off the end.
+struct Cursor<'a> {
+    data: &'a [u8],
+    pos: usize,
+    overrun: FormatError,
+}
+
+impl<'a> Cursor<'a> {
+    fn new(data: &'a [u8], overrun: FormatError) -> Self {
+        Cursor {
+            data,
+            pos: 0,
+            overrun,
+        }
+    }
+
+    fn take(&mut self, n: usize) -> Result<&'a [u8], FormatError> {
+        let end = self
+            .pos
+            .checked_add(n)
+            .filter(|&e| e <= self.data.len())
+            .ok_or_else(|| self.overrun.clone())?;
+        let s = &self.data[self.pos..end];
+        self.pos = end;
+        Ok(s)
+    }
+
+    fn u8(&mut self) -> Result<u8, FormatError> {
+        Ok(self.take(1)?[0])
+    }
+
+    fn uint(&mut self, width: u8) -> Result<u64, FormatError> {
+        let b = self.take(width as usize)?;
+        Ok(b.iter()
+            .rev()
+            .fold(0u64, |acc, &x| (acc << 8) | u64::from(x)))
+    }
+
+    /// An address of `width` bytes; `None` when undefined (all ones).
+    fn addr(&mut self, width: u8) -> Result<Option<u64>, FormatError> {
+        let v = self.uint(width)?;
+        let undef = if width >= 8 {
+            u64::MAX
+        } else {
+            (1u64 << (8 * u32::from(width))) - 1
+        };
+        Ok((v != undef).then_some(v))
+    }
+}
+
+/// Decode and check the superblock extension of `sb`, as libhdf5 does when
+/// it opens the file. `data` is the file from the superblock on, up to the
+/// end of file the superblock records (its end is libhdf5's "eoa").
+///
+/// Returns `Ok(None)` for a superblock without an extension (versions 0
+/// and 1 have none). A message libhdf5 fails to decode, or a cache image
+/// that does not lie inside the file, is an error: libhdf5 refuses to open
+/// such a file (`cve-2020-10810`: a File Space Info message too short for
+/// the free-space manager addresses it announces; `cve-2020-10812`: a cache
+/// image past the end of the file).
+pub fn read_superblock_extension(
+    data: &[u8],
+    sb: &Superblock,
+) -> Result<Option<SuperblockExtension>, FormatError> {
+    read_superblock_extension_in(data, sb)
+}
+
+/// [`read_superblock_extension`] over any [`Storage`]; its length is the
+/// end of file.
+pub fn read_superblock_extension_in<S: Storage + ?Sized>(
+    file: &S,
+    sb: &Superblock,
+) -> Result<Option<SuperblockExtension>, FormatError> {
+    let os = sb.offset_size;
+    let ls = sb.length_size;
+    let undef = if os >= 8 {
+        u64::MAX
+    } else {
+        (1u64 << (8 * u32::from(os))) - 1
+    };
+    let Some(addr) = sb.superblock_extension_address.filter(|&a| a != undef) else {
+        return Ok(None);
+    };
+    let addr = usize::try_from(addr).map_err(|_| ext_err("address out of range"))?;
+    let header = ObjectHeader::parse_in(file, addr as u64, os, ls)?;
+    let eoa = file.len();
+
+    let mut ext = SuperblockExtension::default();
+    for msg in &header.messages {
+        match msg.msg_type {
+            MessageType::BTreeKValues => {
+                let mut c = Cursor::new(&msg.data, ext_err(RAN_OFF));
+                if c.u8()? != 0 {
+                    return Err(ext_err("bad version number for v1 B-tree 'K' message"));
+                }
+                let chunk = c.uint(2)? as u16;
+                let snode = c.uint(2)? as u16;
+                let leaf = c.uint(2)? as u16;
+                ext.btree_k = Some((chunk, snode, leaf));
+            }
+            MessageType::Unknown(MSG_FSINFO) if msg.flags & MSG_FLAG_WAS_UNKNOWN == 0 => {
+                ext.file_space_info = Some(decode_fsinfo(&msg.data, os, ls)?);
+            }
+            MessageType::Unknown(MSG_MDCI) => {
+                let mut c = Cursor::new(&msg.data, ext_err(RAN_OFF));
+                if c.u8()? != 0 {
+                    return Err(ext_err(
+                        "bad version number for metadata cache image message",
+                    ));
+                }
+                let address = c.addr(os)?;
+                let length = c.uint(ls)?;
+                let Some(address) = address else {
+                    return Err(ext_err("metadata cache image address is undefined"));
+                };
+                if address.checked_add(length).is_none_or(|end| end > eoa) {
+                    return Err(ext_err(
+                        "metadata cache image: address plus size exceeds file eoa",
+                    ));
+                }
+                ext.cache_image = Some(CacheImageLocation { address, length });
+            }
+            _ => {}
+        }
+    }
+    Ok(Some(ext))
+}
+
+/// `H5O__fsinfo_decode` plus the checks `H5F__super_read` makes on it.
+fn decode_fsinfo(data: &[u8], os: u8, ls: u8) -> Result<FileSpaceInfo, FormatError> {
+    let mut c = Cursor::new(data, ext_err(RAN_OFF));
+    let version = c.u8()?;
+    let info = if version == 0 {
+        let old_strategy = c.u8()?;
+        let threshold = c.uint(ls)?;
+        // H5F_file_space_type_t: 1 ALL_PERSIST, 2 ALL, 3 AGGR_VFD, 4 VFD.
+        let (strategy, persist) = match old_strategy {
+            1 => {
+                for _ in 0..FSM_ADDRS_V0 {
+                    c.addr(os)?;
+                }
+                (0, true)
+            }
+            2 => (0, false),
+            3 => (2, false),
+            4 => (3, false),
+            _ => return Err(ext_err("invalid file space strategy")),
+        };
+        FileSpaceInfo {
+            version,
+            strategy,
+            persist,
+            threshold,
+            page_size: PAGE_SIZE_DEFAULT,
+        }
+    } else {
+        if version > 1 {
+            return Err(ext_err("File space info message's version out of bounds"));
+        }
+        let strategy = c.u8()?;
+        let persist = c.u8()? != 0;
+        let threshold = c.uint(ls)?;
+        let page_size = c.uint(ls)?;
+        if page_size == 0 || page_size > PAGE_SIZE_MAX {
+            return Err(ext_err("invalid page size in file space info"));
+        }
+        c.uint(2)?; // page end metadata threshold
+        c.addr(os)?; // EOA before the free-space managers
+        if persist {
+            for _ in 0..FSM_ADDRS_V1 {
+                c.addr(os)?;
+            }
+        }
+        FileSpaceInfo {
+            version,
+            strategy,
+            persist,
+            threshold,
+            page_size,
+        }
+    };
+    if info.page_size < PAGE_SIZE_MIN {
+        return Err(ext_err("file space page size too small"));
+    }
+    Ok(info)
+}
+
+/// One entry of a metadata cache image: `len` bytes at `image_offset` in
+/// the image block, belonging at file address `address`.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+struct ImageEntry {
+    address: u64,
+    image_offset: usize,
+    len: usize,
+}
+
+/// A decoded metadata cache image: where its block is, and the entries it
+/// holds. [`CacheImage::apply`] writes the entries over a file's bytes.
+///
+/// Only the entry list is kept, never a copy of the file: an opener that
+/// maps the file applies the image to a private copy-on-write mapping, so
+/// only the pages the entries land on are copied.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct CacheImage {
+    location: CacheImageLocation,
+    entries: Vec<ImageEntry>,
+}
+
+/// What an opener must do about a file's metadata cache image.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum CacheImageState {
+    /// The file has no image: its bytes are its metadata.
+    Absent,
+    /// The file has an image that loads: apply it with [`CacheImage::apply`].
+    Loaded(CacheImage),
+    /// The file has an image libhdf5 fails to load. libhdf5 still opens the
+    /// file (the image loads at the first metadata read), and that read
+    /// fails with this error.
+    Unloadable(FormatError),
+}
+
+impl CacheImage {
+    /// Decode the metadata cache image at `location` in `data` (the file
+    /// from the superblock on, up to its recorded end of file). The image is
+    /// checked as libhdf5 checks it (`H5C__decode_cache_image_header`,
+    /// `H5C__reconstruct_cache_entry`): signature and version, the image
+    /// length it records, entry types, rings and ages in range, entry
+    /// addresses inside the file and not repeated, flush-dependency parents
+    /// already in the cache.
+    ///
+    /// One check is stricter than libhdf5's: an entry must end inside the
+    /// file. libhdf5 checks only that it starts there, and serves the rest
+    /// from the image; the images libhdf5 writes never do this (every entry
+    /// lies below the image block, which is written last), and the bytes an
+    /// entry would put past the end of file have nowhere to go in a view of
+    /// the file.
+    ///
+    /// libhdf5 does not verify the block's trailing checksum when it loads
+    /// an image, so neither does this.
+    pub fn decode(
+        data: &[u8],
+        location: CacheImageLocation,
+        sb: &Superblock,
+    ) -> Result<Self, FormatError> {
+        Self::decode_in(data, location, sb)
+    }
+
+    /// [`Self::decode`] over any [`Storage`]: one read of the image block.
+    pub fn decode_in<S: Storage + ?Sized>(
+        file: &S,
+        location: CacheImageLocation,
+        sb: &Superblock,
+    ) -> Result<Self, FormatError> {
+        let (offset_size, length_size) = (sb.offset_size, sb.length_size);
+        let bad = FormatError::InvalidCacheImage;
+        let block = image_block_in(file, location)?;
+        let eoa = file.len();
+        let mut c = Cursor::new(&block, bad(RAN_OFF));
+
+        // Header: signature, version, flags, image data length, entry count.
+        if c.take(4)? != MDCI_SIGNATURE {
+            return Err(bad("bad metadata cache image header signature"));
+        }
+        if c.u8()? != 0 {
+            return Err(bad("bad metadata cache image version"));
+        }
+        if c.u8()? & MDCI_HAVE_RESIZE_STATUS != 0 {
+            return Err(bad("MDC resize status not yet supported"));
+        }
+        if c.uint(length_size)? != location.length {
+            return Err(bad("bad metadata cache image data length"));
+        }
+        let n_entries = c.uint(4)?;
+        if n_entries == 0 {
+            return Err(bad("bad metadata cache entry count"));
+        }
+
+        let mut entries = Vec::new();
+        // What is in libhdf5's cache when it loads the image: the superblock
+        // and the superblock extension's object header (read to find the
+        // image). Each entry's flush-dependency parents are looked up in the
+        // cache as the entry is inserted (`H5C__reconstruct_cache_contents`
+        // searches the index inside the loop that inserts the entries, in
+        // HDF5 1.14.6 and 2.0.0 alike), so a parent must be one of those or
+        // an earlier entry.
+        let mut cached = BTreeSet::new();
+        cached.insert(0);
+        if let Some(ext) = sb.superblock_extension_address {
+            cached.insert(ext);
+        }
+        let mut seen = BTreeSet::new();
+        for _ in 0..n_entries {
+            let type_id = c.u8()?;
+            if type_id >= MDCI_NTYPES {
+                return Err(bad("type id is out of valid range"));
+            }
+            let flags = c.u8()?;
+            if c.u8()? >= MDCI_RING_NTYPES {
+                return Err(bad("ring is out of valid range"));
+            }
+            if c.u8()? > MDCI_AGE_MAX {
+                return Err(bad("entry age is out of policy range"));
+            }
+            let children = c.uint(2)?;
+            // libhdf5 checks the parent flag against the child count only in
+            // debug builds (release builds refuse any entry with children);
+            // the image format's own rule is checked here.
+            if (flags & MDCI_ENTRY_IS_FD_PARENT != 0) != (children > 0) {
+                return Err(bad("flush dependency parent flag and child count disagree"));
+            }
+            c.uint(2)?; // dirty dependency children: reset for a read-only open
+            let parents = c.uint(2)?;
+            if (flags & MDCI_ENTRY_IS_FD_CHILD != 0) != (parents > 0) {
+                return Err(bad("flush dependency child flag and parent count disagree"));
+            }
+            c.uint(4)?; // LRU rank
+            let address = c
+                .addr(offset_size)?
+                .filter(|&a| a < eoa)
+                .ok_or(bad("invalid entry address range"))?;
+            let size = c.uint(length_size)?;
+            if size == 0 {
+                return Err(bad("invalid entry size"));
+            }
+            for _ in 0..parents {
+                let parent = c
+                    .addr(offset_size)?
+                    .ok_or(bad("invalid flush dependency parent offset"))?;
+                if !seen.contains(&parent) && !cached.contains(&parent) {
+                    return Err(bad("fd parent not in cache"));
+                }
+            }
+            let len = usize::try_from(size).map_err(|_| bad(RAN_OFF))?;
+            let image_offset = c.pos;
+            c.take(len)?;
+            if address.checked_add(size).is_none_or(|end| end > eoa) {
+                return Err(bad("entry extends past the end of file"));
+            }
+            if !seen.insert(address) {
+                return Err(bad("duplicate addresses in cache"));
+            }
+            entries.push(ImageEntry {
+                address,
+                image_offset,
+                len,
+            });
+        }
+        Ok(CacheImage { location, entries })
+    }
+
+    /// Where the image block is.
+    pub fn location(&self) -> CacheImageLocation {
+        self.location
+    }
+
+    /// The number of entries in the image.
+    pub fn len(&self) -> usize {
+        self.entries.len()
+    }
+
+    /// Whether the image has no entries (a decoded image always has some).
+    pub fn is_empty(&self) -> bool {
+        self.entries.is_empty()
+    }
+
+    /// The file ranges (address, length) the image's entries replace.
+    pub fn entry_ranges(&self) -> impl Iterator<Item = (u64, usize)> + '_ {
+        self.entries.iter().map(|e| (e.address, e.len))
+    }
+
+    /// The image block in `data`, the bytes [`Self::decode`] read it from.
+    pub fn block<'a>(&self, data: &'a [u8]) -> Result<&'a [u8], FormatError> {
+        image_block(data, self.location)
+    }
+
+    /// [`Self::block`] over any [`Storage`].
+    pub fn block_in<'a, S: Storage + ?Sized>(
+        &self,
+        file: &'a S,
+    ) -> Result<Cow<'a, [u8]>, FormatError> {
+        image_block_in(file, self.location)
+    }
+
+    /// Every entry as `(file address, its bytes)`, taken from `block` (the
+    /// image block, see [`Self::block_in`]), in the order [`Self::apply`]
+    /// writes them: for a reader that cannot write the image over the
+    /// file's bytes and lays the entries over each read instead.
+    pub fn entries<'b>(&self, block: &'b [u8]) -> Result<Vec<(u64, &'b [u8])>, FormatError> {
+        let short = || FormatError::InvalidCacheImage("image applied to the wrong file");
+        self.entries
+            .iter()
+            .map(|e| {
+                let src = block
+                    .get(e.image_offset..e.image_offset + e.len)
+                    .ok_or_else(short)?;
+                Ok((e.address, src))
+            })
+            .collect()
+    }
+
+    /// Write every entry over `dst`, the file's bytes from the superblock
+    /// on (as long as the `data` the image was decoded from), taking the
+    /// entries from `block` (the image block, see [`Self::block`]). `block`
+    /// must not alias `dst`: an entry may land on the block itself.
+    pub fn apply(&self, block: &[u8], dst: &mut [u8]) -> Result<(), FormatError> {
+        let short = || FormatError::InvalidCacheImage("image applied to the wrong file");
+        for e in &self.entries {
+            let src = block
+                .get(e.image_offset..e.image_offset + e.len)
+                .ok_or_else(short)?;
+            let at = usize::try_from(e.address).map_err(|_| short())?;
+            dst.get_mut(at..at + e.len)
+                .ok_or_else(short)?
+                .copy_from_slice(src);
+        }
+        Ok(())
+    }
+}
+
+fn image_block(data: &[u8], location: CacheImageLocation) -> Result<&[u8], FormatError> {
+    let (start, len) = image_block_range(data.len() as u64, location)?;
+    let start = crate::addr::to_usize(start)?;
+    Ok(&data[start..start + len])
+}
+
+fn image_block_in<S: Storage + ?Sized>(
+    file: &S,
+    location: CacheImageLocation,
+) -> Result<Cow<'_, [u8]>, FormatError> {
+    let (start, len) = image_block_range(file.len(), location)?;
+    read_exact_at(file, start, len)
+}
+
+/// Where the image block is, checked against a file of `file_len` bytes.
+fn image_block_range(
+    file_len: u64,
+    location: CacheImageLocation,
+) -> Result<(u64, usize), FormatError> {
+    let bad = FormatError::InvalidCacheImage;
+    let start = usize::try_from(location.address).map_err(|_| bad("address out of range"))?;
+    let len = usize::try_from(location.length).map_err(|_| bad("length out of range"))?;
+    start
+        .checked_add(len)
+        .filter(|&end| end as u64 <= file_len)
+        .ok_or(bad("image block extends past the end of the file"))?;
+    Ok((start as u64, len))
+}
+
+/// What an opener must do before reading a file's metadata: check the
+/// superblock extension ([`read_superblock_extension`]; an error means
+/// libhdf5 refuses to open the file) and decode any metadata cache image
+/// ([`CacheImage::decode`]). `data` is the file from the superblock on, up
+/// to its recorded end of file.
+pub fn cache_image_state(data: &[u8], sb: &Superblock) -> Result<CacheImageState, FormatError> {
+    cache_image_state_in(data, sb)
+}
+
+/// [`cache_image_state`] over any [`Storage`].
+pub fn cache_image_state_in<S: Storage + ?Sized>(
+    file: &S,
+    sb: &Superblock,
+) -> Result<CacheImageState, FormatError> {
+    match read_superblock_extension_in(file, sb)? {
+        Some(SuperblockExtension {
+            cache_image: Some(location),
+            ..
+        }) => Ok(match CacheImage::decode_in(file, location, sb) {
+            Ok(image) => CacheImageState::Loaded(image),
+            Err(e) => CacheImageState::Unloadable(e),
+        }),
+        _ => Ok(CacheImageState::Absent),
+    }
+}
+
+/// [`cache_image_state`] for a reader that holds the file's bytes in a
+/// buffer of its own: check the superblock extension and write any cache
+/// image over `data` in place (only the image block is copied). An image
+/// libhdf5 cannot load is an error here: such a reader has no way to open
+/// the file and fail each object instead.
+pub fn apply_cache_image_in_place(data: &mut [u8], sb: &Superblock) -> Result<(), FormatError> {
+    match cache_image_state(data, sb)? {
+        CacheImageState::Absent => Ok(()),
+        CacheImageState::Unloadable(e) => Err(e),
+        CacheImageState::Loaded(image) => {
+            let block = image.block(data)?.to_vec();
+            image.apply(&block, data)
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    /// The file's bytes with the image at `loc` applied.
+    fn apply_cache_image(
+        data: &[u8],
+        loc: CacheImageLocation,
+        sb: &Superblock,
+    ) -> Result<Vec<u8>, FormatError> {
+        let image = CacheImage::decode(data, loc, sb)?;
+        let mut out = data.to_vec();
+        image.apply(image.block(data)?, &mut out)?;
+        Ok(out)
+    }
+
+    fn sb_v2(ext: u64) -> Superblock {
+        Superblock {
+            version: 2,
+            offset_size: 8,
+            length_size: 8,
+            base_address: 0,
+            eof_address: 0,
+            root_group_address: 0,
+            group_leaf_node_k: None,
+            group_internal_node_k: None,
+            indexed_storage_internal_node_k: None,
+            free_space_address: None,
+            driver_info_address: None,
+            consistency_flags: 0,
+            superblock_extension_address: Some(ext),
+            checksum: None,
+            page_size: None,
+        }
+    }
+
+    /// A file whose superblock extension (a version 1 object header at 48)
+    /// holds the given messages, padded to `len` bytes.
+    fn file_with_ext(messages: &[(u16, &[u8])], len: usize) -> Vec<u8> {
+        let mut body = Vec::new();
+        for &(t, d) in messages {
+            let padded = d.len().div_ceil(8) * 8;
+            body.extend_from_slice(&t.to_le_bytes());
+            body.extend_from_slice(&(padded as u16).to_le_bytes());
+            body.extend_from_slice(&[0x14, 0, 0, 0]);
+            body.extend_from_slice(d);
+            body.resize(body.len() + padded - d.len(), 0);
+        }
+        let mut f = vec![0u8; 48];
+        f.push(1);
+        f.push(0);
+        f.extend_from_slice(&(messages.len() as u16).to_le_bytes());
+        f.extend_from_slice(&1u32.to_le_bytes());
+        f.extend_from_slice(&(body.len() as u32).to_le_bytes());
+        f.extend_from_slice(&[0; 4]);
+        f.extend_from_slice(&body);
+        f.resize(len, 0);
+        f
+    }
+
+    fn fsinfo_v1(page_size: u64, persist: bool, n_addrs: usize) -> Vec<u8> {
+        let mut m = vec![1, 1, u8::from(persist)];
+        m.extend_from_slice(&1u64.to_le_bytes());
+        m.extend_from_slice(&page_size.to_le_bytes());
+        m.extend_from_slice(&0u16.to_le_bytes());
+        m.extend_from_slice(&u64::MAX.to_le_bytes());
+        for _ in 0..n_addrs {
+            m.extend_from_slice(&u64::MAX.to_le_bytes());
+        }
+        m
+    }
+
+    fn mdci(address: u64, length: u64) -> Vec<u8> {
+        let mut m = vec![0];
+        m.extend_from_slice(&address.to_le_bytes());
+        m.extend_from_slice(&length.to_le_bytes());
+        m
+    }
+
+    #[test]
+    fn no_extension() {
+        assert_eq!(
+            read_superblock_extension(&[0; 64], &sb_v2(u64::MAX)).unwrap(),
+            None
+        );
+    }
+
+    #[test]
+    fn file_space_info_as_libhdf5_decodes_it() {
+        // What FileWriter::with_page_size writes.
+        let f = file_with_ext(&[(MSG_FSINFO, &fsinfo_v1(4096, false, 0))], 256);
+        let ext = read_superblock_extension(&f, &sb_v2(48)).unwrap().unwrap();
+        assert_eq!(ext.file_space_info.unwrap().page_size, 4096);
+        let f = file_with_ext(&[(MSG_FSINFO, &fsinfo_v1(4096, true, 12))], 512);
+        assert!(read_superblock_extension(&f, &sb_v2(48)).is_ok());
+
+        let refused = |m: Vec<u8>| {
+            let f = file_with_ext(&[(MSG_FSINFO, &m)], 512);
+            read_superblock_extension(&f, &sb_v2(48)).unwrap_err()
+        };
+        // Persisting, but too short for the manager addresses.
+        let mut short = fsinfo_v1(4096, true, 12);
+        short.truncate(short.len() - 8);
+        assert_eq!(refused(short), ext_err(RAN_OFF));
+        assert!(matches!(
+            refused(fsinfo_v1(256, false, 0)),
+            FormatError::InvalidSuperblockExtension(_)
+        ));
+        assert!(matches!(
+            refused(fsinfo_v1(0, false, 0)),
+            FormatError::InvalidSuperblockExtension(_)
+        ));
+        let mut v2 = fsinfo_v1(4096, false, 0);
+        v2[0] = 2;
+        assert!(matches!(
+            refused(v2),
+            FormatError::InvalidSuperblockExtension(_)
+        ));
+        // cve-2020-10810: version 0, strategy ALL_PERSIST, and a message of
+        // 32 bytes that cannot hold the six addresses that follow.
+        let mut v0 = vec![0u8, 1];
+        v0.extend_from_slice(&[0, 1, 0, 0, 0, 0, 0, 0]);
+        v0.resize(32, 0xff);
+        assert_eq!(refused(v0), ext_err(RAN_OFF));
+        // A version 0 message without persistence is fine.
+        let mut v0 = vec![0u8, 2];
+        v0.extend_from_slice(&[0; 8]);
+        let f = file_with_ext(&[(MSG_FSINFO, &v0)], 256);
+        assert!(read_superblock_extension(&f, &sb_v2(48)).is_ok());
+    }
+
+    #[test]
+    fn cache_image_location_must_be_inside_the_file() {
+        let f = file_with_ext(&[(MSG_MDCI, &mdci(128, 64))], 192);
+        let ext = read_superblock_extension(&f, &sb_v2(48)).unwrap().unwrap();
+        assert_eq!(
+            ext.cache_image,
+            Some(CacheImageLocation {
+                address: 128,
+                length: 64
+            })
+        );
+        // cve-2020-10812: 256 MiB at 0x10100 in a 2565-byte file.
+        let f = file_with_ext(&[(MSG_MDCI, &mdci(0x10100, 0x1000_0000))], 2565);
+        assert!(matches!(
+            read_superblock_extension(&f, &sb_v2(48)),
+            Err(FormatError::InvalidSuperblockExtension(_))
+        ));
+        let f = file_with_ext(&[(MSG_MDCI, &mdci(u64::MAX, 8))], 256);
+        assert!(read_superblock_extension(&f, &sb_v2(48)).is_err());
+    }
+
+    /// A cache image block with `entries` of (address, bytes).
+    fn image(entries: &[(u64, &[u8])]) -> Vec<u8> {
+        let with_deps: Vec<_> = entries.iter().map(|&(a, b)| (a, b, 0, None)).collect();
+        image_with_deps(&with_deps)
+    }
+
+    /// A cache image block with `entries` of (address, bytes, flush
+    /// dependency children, flush dependency parent).
+    fn image_with_deps(entries: &[(u64, &[u8], u16, Option<u64>)]) -> Vec<u8> {
+        let mut b = Vec::new();
+        b.extend_from_slice(MDCI_SIGNATURE);
+        b.push(0);
+        b.push(0);
+        b.extend_from_slice(&0u64.to_le_bytes()); // length, patched below
+        b.extend_from_slice(&(entries.len() as u32).to_le_bytes());
+        for &(addr, bytes, children, parent) in entries {
+            let mut flags = 0x02; // in LRU
+            if children > 0 {
+                flags |= MDCI_ENTRY_IS_FD_PARENT;
+            }
+            if parent.is_some() {
+                flags |= MDCI_ENTRY_IS_FD_CHILD;
+            }
+            b.extend_from_slice(&[5, flags, 1, 0]); // type, flags, ring, age
+            b.extend_from_slice(&children.to_le_bytes());
+            b.extend_from_slice(&0u16.to_le_bytes()); // dirty children
+            b.extend_from_slice(&u16::from(parent.is_some()).to_le_bytes());
+            b.extend_from_slice(&0i32.to_le_bytes());
+            b.extend_from_slice(&addr.to_le_bytes());
+            b.extend_from_slice(&(bytes.len() as u64).to_le_bytes());
+            if let Some(p) = parent {
+                b.extend_from_slice(&p.to_le_bytes());
+            }
+            b.extend_from_slice(bytes);
+        }
+        b.extend_from_slice(&[0; 4]); // checksum (not verified, as in libhdf5)
+        let n = b.len() as u64;
+        b[6..14].copy_from_slice(&n.to_le_bytes());
+        b
+    }
+
+    #[test]
+    fn cache_image_entries_replace_the_file_bytes() {
+        let img = image(&[(16, b"HEADER"), (40, b"NODE")]);
+        let mut f = vec![0u8; 64];
+        let at = f.len() as u64;
+        f.extend_from_slice(&img);
+        let loc = CacheImageLocation {
+            address: at,
+            length: img.len() as u64,
+        };
+        let out = apply_cache_image(&f, loc, &sb_v2(u64::MAX)).unwrap();
+        assert_eq!(out.len(), f.len());
+        assert_eq!(&out[16..22], b"HEADER");
+        assert_eq!(&out[40..44], b"NODE");
+        assert_eq!(&out[..16], &f[..16]);
+
+        let bad = |img: Vec<u8>| {
+            let mut f = vec![0u8; 64];
+            f.extend_from_slice(&img);
+            let loc = CacheImageLocation {
+                address: 64,
+                length: img.len() as u64,
+            };
+            apply_cache_image(&f, loc, &sb_v2(u64::MAX)).unwrap_err()
+        };
+        let mut sig = image(&[(16, b"x")]);
+        sig[0] = b'X';
+        assert!(matches!(bad(sig), FormatError::InvalidCacheImage(_)));
+        assert!(matches!(
+            bad(image(&[(16, b"a"), (16, b"b")])),
+            FormatError::InvalidCacheImage("duplicate addresses in cache")
+        ));
+        assert!(matches!(
+            bad(image(&[(1 << 20, b"far")])),
+            FormatError::InvalidCacheImage("invalid entry address range")
+        ));
+        let mut len = image(&[(16, b"x")]);
+        len[6] ^= 1;
+        assert!(matches!(bad(len), FormatError::InvalidCacheImage(_)));
+        // An entry that starts inside the file (64 bytes, then a 60-byte
+        // image) but runs past its end.
+        assert!(matches!(
+            bad(image(&[(123, b"8 bytes!")])),
+            FormatError::InvalidCacheImage("entry extends past the end of file")
+        ));
+        let mut cut = image(&[(16, b"abcdef")]);
+        let n = cut.len() as u64 - 8;
+        cut.truncate(cut.len() - 8);
+        cut[6..14].copy_from_slice(&n.to_le_bytes());
+        assert!(matches!(bad(cut), FormatError::InvalidCacheImage(_)));
+    }
+
+    /// libhdf5 resolves an entry's flush-dependency parents as it inserts
+    /// the entry (`H5C__reconstruct_cache_contents`): a parent must be an
+    /// earlier entry, or the superblock or its extension's object header,
+    /// which are cached before the image loads. A parent listed after its
+    /// child fails ("fd parent not in cache?!?").
+    #[test]
+    fn flush_dependency_parents_must_already_be_cached() {
+        let load = |img: Vec<u8>| {
+            let mut f = vec![0u8; 64];
+            f.extend_from_slice(&img);
+            let loc = CacheImageLocation {
+                address: 64,
+                length: img.len() as u64,
+            };
+            apply_cache_image(&f, loc, &sb_v2(48))
+        };
+        // Parent first, as libhdf5 writes images.
+        assert!(
+            load(image_with_deps(&[
+                (16, b"P", 1, None),
+                (40, b"C", 0, Some(16))
+            ]))
+            .is_ok()
+        );
+        // Child first: libhdf5 does not find the parent.
+        assert_eq!(
+            load(image_with_deps(&[
+                (40, b"C", 0, Some(16)),
+                (16, b"P", 1, None)
+            ]))
+            .unwrap_err(),
+            FormatError::InvalidCacheImage("fd parent not in cache")
+        );
+        // The superblock extension's header (at 48 here) is in the cache.
+        assert!(load(image_with_deps(&[(40, b"C", 0, Some(48))])).is_ok());
+        // An entry cannot be its own parent.
+        assert!(load(image_with_deps(&[(40, b"C", 1, Some(40))])).is_err());
+    }
+
+    /// The extension and cache image decode identically through a
+    /// `read_at`-only storage, errors included.
+    #[test]
+    fn storage_parse_matches_slice_parse() {
+        use crate::storage::CountingStorage;
+        let img = image(&[(16, b"HEADER"), (40, b"NODE")]);
+        let mut with_image = file_with_ext(&[(MSG_MDCI, &mdci(256, img.len() as u64))], 256);
+        with_image.extend_from_slice(&img);
+        let mut bad_image = with_image.clone();
+        bad_image[256] = b'X';
+        let files = [
+            file_with_ext(&[(MSG_FSINFO, &fsinfo_v1(4096, false, 0))], 256),
+            file_with_ext(&[(MSG_FSINFO, &fsinfo_v1(256, false, 0))], 256),
+            file_with_ext(&[(MSG_MDCI, &mdci(128, 64))], 192),
+            file_with_ext(&[(MSG_MDCI, &mdci(0x10100, 0x1000_0000))], 2565),
+            file_with_ext(&[(MSG_FSINFO, &fsinfo_v1(4096, true, 12))], 60),
+            with_image,
+            bad_image,
+        ];
+        for f in files {
+            let storage = CountingStorage::new(f.clone());
+            let sb = sb_v2(48);
+            assert_eq!(
+                read_superblock_extension_in(&storage, &sb),
+                read_superblock_extension(&f, &sb)
+            );
+            assert_eq!(
+                cache_image_state_in(&storage, &sb),
+                cache_image_state(&f, &sb)
+            );
+            if let Ok(CacheImageState::Loaded(image)) = cache_image_state(&f, &sb) {
+                assert_eq!(
+                    &*image.block_in(&storage).unwrap(),
+                    image.block(&f).unwrap()
+                );
+            }
+        }
+    }
+}

crates/clawhdf5-format/src/symbol_table.rs

@@ -4,6 +4,7 @@
 use alloc::vec::Vec;
 
 use crate::error::FormatError;
+use crate::storage::{Storage, read_exact_at};
 
 /// Symbol Table message (type 0x0011) found in v1 group object headers.
 #[derive(Debug, Clone, PartialEq)]
@@ -79,65 +80,53 @@ impl SymbolTableNode {
         offset: usize,
         offset_size: u8,
     ) -> Result<SymbolTableNode, FormatError> {
-        // signature(4) + version(1) + reserved(1) + number_of_symbols(2) = 8
+        Self::parse_in(file_data, offset as u64, offset_size)
-        if offset
-            .checked_add(8)
-            .is_none_or(|end| end > file_data.len())
-        {
-            return Err(FormatError::UnexpectedEof {
-                expected: offset.saturating_add(8),
-                available: file_data.len(),
-            });
     }
 
-        if &file_data[offset..offset + 4] != b"SNOD" {
+    /// [`Self::parse`] over any [`Storage`]: one read of the node's header,
+    /// one of its entries.
+    pub fn parse_in<S: Storage + ?Sized>(
+        file: &S,
+        offset: u64,
+        offset_size: u8,
+    ) -> Result<SymbolTableNode, FormatError> {
+        // The entries are read once the header says how many there are:
+        // say so (see `Storage::hint`), for libhdf5's default node size
+        // (group leaf K = 4: 8 entries of 40 bytes with 8-byte offsets).
+        file.hint(offset, 8 + 8 * (2 * usize::from(offset_size) + 24));
+        // signature(4) + version(1) + reserved(1) + number_of_symbols(2) = 8
+        let header = read_exact_at(file, offset, 8)?;
+
+        if &header[..4] != b"SNOD" {
             return Err(FormatError::InvalidSymbolTableNodeSignature);
         }
 
-        let version = file_data[offset + 4];
+        let version = header[4];
         if version != 1 {
             return Err(FormatError::InvalidSymbolTableNodeVersion(version));
         }
 
-        let num_symbols =
+        let num_symbols = u16::from_le_bytes([header[6], header[7]]) as usize;
-            u16::from_le_bytes([file_data[offset + 6], file_data[offset + 7]]) as usize;
 
         let os = offset_size as usize;
         // Each entry: link_name_offset(os) + obj_hdr_addr(os) + cache_type(4) + reserved(4) + scratch(16)
         let entry_size = os + os + 4 + 4 + 16;
-        let entries_start = offset + 8;
+        // `offset + 8` fits: the header's read checked it. The entries'
-        let needed = entries_start.checked_add(num_symbols * entry_size).ok_or(
+        // read is the bounds check (`offset + 8 + entries > file length`,
-            FormatError::UnexpectedEof {
+        // which cannot overflow: at most 65535 entries of 40 bytes).
-                expected: usize::MAX,
+        let body = read_exact_at(file, offset + 8, num_symbols * entry_size)?;
-                available: file_data.len(),
+        let file_data: &[u8] = &body;
-            },
-        )?;
-        if needed > file_data.len() {
-            return Err(FormatError::UnexpectedEof {
-                expected: needed,
-                available: file_data.len(),
-            });
-        }
 
         let mut entries = Vec::with_capacity(num_symbols);
-        let mut pos = entries_start;
+        for entry in file_data.chunks_exact(entry_size) {
-        for _ in 0..num_symbols {
+            let link_name_offset = read_offset(entry, 0, offset_size)?;
-            let link_name_offset = read_offset(file_data, pos, offset_size)?;
+            let object_header_address = read_offset(entry, os, offset_size)?;
-            pos += os;
+            let pos = 2 * os;
-            let object_header_address = read_offset(file_data, pos, offset_size)?;
+            let cache_type =
-            pos += os;
+                u32::from_le_bytes([entry[pos], entry[pos + 1], entry[pos + 2], entry[pos + 3]]);
-            let cache_type = u32::from_le_bytes([
-                file_data[pos],
-                file_data[pos + 1],
-                file_data[pos + 2],
-                file_data[pos + 3],
-            ]);
-            pos += 4;
             // reserved 4 bytes
-            pos += 4;
             let mut scratch_pad = [0u8; 16];
-            scratch_pad.copy_from_slice(&file_data[pos..pos + 16]);
+            scratch_pad.copy_from_slice(&entry[pos + 8..pos + 24]);
-            pos += 16;
 
             entries.push(SymbolTableEntry {
                 link_name_offset,
@@ -256,4 +245,28 @@ mod tests {
         let result = SymbolTableNode::parse(&data, usize::MAX / 2, 8);
         assert!(result.is_err());
     }
+
+    /// Nodes, cut at every length and at an offset, parse identically
+    /// through a `read_at`-only storage.
+    #[test]
+    fn storage_parse_matches_slice_parse() {
+        use crate::storage::CountingStorage;
+        for os in [4u8, 8] {
+            let node = build_snod(&[(0, 0x100, 0), (8, 0x200, 1), (16, 0x300, 2)], os);
+            let mut bad = node.clone();
+            bad[4] = 2;
+            for full in [node, bad] {
+                for at in [0usize, 7] {
+                    for cut in 0..=full.len() {
+                        let mut f = vec![0u8; at];
+                        f.extend_from_slice(&full[..cut]);
+                        let storage = CountingStorage::new(f.clone());
+                        let want = SymbolTableNode::parse(&f, at, os);
+                        let got = SymbolTableNode::parse_in(&storage, at as u64, os);
+                        assert_eq!(format!("{got:?}"), format!("{want:?}"));
+                    }
+                }
+            }
+        }
+    }
 }

crates/clawhdf5-format/src/type_builders.rs

@@ -137,6 +137,37 @@ pub fn make_f32_type() -> Datatype {
     }
 }
 
+/// numpy `complex64` the way h5py stores it: a compound `{r: f32, i: f32}`.
+/// Every HDF5 reader opens it; h5py reads it back as `complex64`.
+pub fn make_complex_f32_type() -> Datatype {
+    Datatype::complex_as_compound(8, &make_f32_type())
+}
+
+/// numpy `complex128` the way h5py stores it: a compound `{r: f64, i: f64}`.
+pub fn make_complex_f64_type() -> Datatype {
+    Datatype::complex_as_compound(16, &make_f64_type())
+}
+
+/// HDF5 2.0's native complex type `H5T_COMPLEX_IEEE_F32LE` (datatype class
+/// 11). Only libhdf5 2.0 and newer (h5py built on it) can read a file that
+/// uses it; older libhdf5, including h5dump 1.14, refuses the object.
+/// Prefer [`make_complex_f32_type`] unless the consumer wants class 11.
+pub fn make_native_complex_f32_type() -> Datatype {
+    Datatype::Complex {
+        size: 8,
+        base_type: Box::new(make_f32_type()),
+    }
+}
+
+/// HDF5 2.0's native complex type `H5T_COMPLEX_IEEE_F64LE` (datatype class
+/// 11); see [`make_native_complex_f32_type`] for who can read it.
+pub fn make_native_complex_f64_type() -> Datatype {
+    Datatype::Complex {
+        size: 16,
+        base_type: Box::new(make_f64_type()),
+    }
+}
+
 pub fn make_i32_type() -> Datatype {
     Datatype::FixedPoint {
         size: 4,
@@ -296,7 +327,8 @@ impl EnumTypeBuilder {
 
 // ---- Attribute helper ----
 
-pub(crate) fn build_attr_message(name: &str, value: &AttrValue) -> AttributeMessage {
+/// The attribute message the writers store for `value` under `name`.
+pub fn build_attr_message(name: &str, value: &AttrValue) -> AttributeMessage {
     match value {
         AttrValue::F64(v) => AttributeMessage {
             name: name.to_string(),
@@ -503,6 +535,9 @@ pub struct DatasetBuilder {
     /// `data` field is ignored; instead the global heap blob is built from
     /// these mappings and a VDS layout message is emitted.
     pub(crate) virtual_sources: Option<Vec<VdsMapping>>,
+    /// Track (and index) attribute creation order; `None` follows the
+    /// file's default (`FileWriter::track_order`).
+    pub(crate) track_order: Option<bool>,
     #[cfg(feature = "provenance")]
     pub(crate) provenance: Option<ProvenanceConfig>,
 }
@@ -522,11 +557,22 @@ impl DatasetBuilder {
             compact: false,
             alignment: 0,
             virtual_sources: None,
+            track_order: None,
             #[cfg(feature = "provenance")]
             provenance: None,
         }
     }
 
+    /// Track the creation order of this dataset's attributes, and index it,
+    /// as h5py's `create_dataset(..., track_order=True)` does: libhdf5 (and
+    /// h5py) then list the attributes in the order they were set rather
+    /// than by name. libhdf5 numbers at most 65 535 attributes on an object
+    /// that tracks their order; more is an error when the file is written.
+    pub fn track_order(&mut self, track: bool) -> &mut Self {
+        self.track_order = Some(track);
+        self
+    }
+
     pub fn with_f64_data(&mut self, data: &[f64]) -> &mut Self {
         self.datatype = Some(make_f64_type());
         let mut b = Vec::with_capacity(data.len() * 8);
@@ -625,6 +671,70 @@ impl DatasetBuilder {
         self
     }
 
+    /// Store complex numbers, each `[re, im]`, as h5py does for numpy
+    /// `complex64`: a compound `{r, i}` of `f32` ([`make_complex_f32_type`]),
+    /// readable by every HDF5 library. For HDF5 2.0's native complex type use
+    /// [`Self::with_native_complex_f32_data`].
+    pub fn with_complex_f32_data(&mut self, data: &[[f32; 2]]) -> &mut Self {
+        self.set_complex(
+            make_complex_f32_type(),
+            data.as_flattened(),
+            f32::to_le_bytes,
+        )
+    }
+
+    /// Store complex numbers, each `[re, im]`, as h5py does for numpy
+    /// `complex128`: a compound `{r, i}` of `f64` ([`make_complex_f64_type`]).
+    pub fn with_complex_f64_data(&mut self, data: &[[f64; 2]]) -> &mut Self {
+        self.set_complex(
+            make_complex_f64_type(),
+            data.as_flattened(),
+            f64::to_le_bytes,
+        )
+    }
+
+    /// Store complex numbers, each `[re, im]`, as HDF5 2.0's native complex
+    /// type `H5T_COMPLEX_IEEE_F32LE` (datatype class 11). The bytes are the
+    /// same as [`Self::with_complex_f32_data`]; only the datatype differs.
+    /// h5py on libhdf5 2.0+ reads it as `complex64`; libhdf5 1.x cannot open
+    /// the dataset at all, so this is opt-in.
+    pub fn with_native_complex_f32_data(&mut self, data: &[[f32; 2]]) -> &mut Self {
+        self.set_complex(
+            make_native_complex_f32_type(),
+            data.as_flattened(),
+            f32::to_le_bytes,
+        )
+    }
+
+    /// Store complex numbers, each `[re, im]`, as HDF5 2.0's native complex
+    /// type `H5T_COMPLEX_IEEE_F64LE` (class 11); see
+    /// [`Self::with_native_complex_f32_data`].
+    pub fn with_native_complex_f64_data(&mut self, data: &[[f64; 2]]) -> &mut Self {
+        self.set_complex(
+            make_native_complex_f64_type(),
+            data.as_flattened(),
+            f64::to_le_bytes,
+        )
+    }
+
+    fn set_complex<T: Copy, const N: usize>(
+        &mut self,
+        datatype: Datatype,
+        parts: &[T],
+        le: fn(T) -> [u8; N],
+    ) -> &mut Self {
+        self.datatype = Some(datatype);
+        let mut b = Vec::with_capacity(parts.len() * N);
+        for &v in parts {
+            b.extend_from_slice(&le(v));
+        }
+        self.data = Some(b);
+        if self.shape.is_none() {
+            self.shape = Some(vec![(parts.len() / 2) as u64]);
+        }
+        self
+    }
+
     /// Write a compound (struct) dataset.
     pub fn with_compound_data(
         &mut self,
@@ -986,10 +1096,11 @@ impl GroupBuilder {
         self.attrs.push((name.to_string(), value));
     }
 
-    /// Track the creation order of this group's links, and index it, as
+    /// Track the creation order of this group's links and attributes, and
-    /// h5py's `track_order=True` does: libhdf5 (and h5py) then list the
+    /// index it, as h5py's `track_order=True` does: libhdf5 (and h5py) then
-    /// group's members in the order they were added rather than by name.
+    /// list the group's members, and its attributes, in the order they were
-    /// Applies to links only, not to attributes.
+    /// added rather than by name. libhdf5 numbers at most 65 535 attributes
+    /// on an object that tracks their order.
     pub fn track_order(&mut self, track: bool) -> &mut Self {
         self.track_order = Some(track);
         self

crates/clawhdf5-format/src/vds.rs

@@ -15,11 +15,13 @@
 #[cfg(not(feature = "std"))]
 use alloc::{format, string::String, vec, vec::Vec};
 
+use crate::addr::{checked_addr, to_usize};
 use crate::data_layout::{DataLayout, VdsMapping, parse_vds_mappings};
 use crate::dataspace::Dataspace;
 use crate::datatype::Datatype;
 use crate::error::FormatError;
 use crate::selection::{SerializedSelection, UNLIMITED};
+use crate::storage::Storage;
 
 /// Resolves the name of an external VDS source file, as stored in the
 /// mapping, to that file's bytes.
@@ -191,8 +193,8 @@ fn non_unlimited_elements(sel: &SerializedSelection, skip: usize) -> Option<u64>
 }
 
 /// Load and decode the mapping list of a virtual layout.
-fn load_mappings(
+fn load_mappings<S: Storage + ?Sized>(
-    file_data: &[u8],
+    file_data: &S,
     layout: &DataLayout,
     length_size: u8,
 ) -> Result<Vec<Mapping>, FormatError> {
@@ -207,8 +209,11 @@ fn load_mappings(
     let Some(addr) = *global_heap_address else {
         return Ok(Vec::new());
     };
-    let coll =
+    let coll = crate::global_heap::GlobalHeapCollection::parse_in(
-        crate::global_heap::GlobalHeapCollection::parse(file_data, addr as usize, length_size)?;
+        file_data,
+        checked_addr(addr)?,
+        length_size,
+    )?;
     let index = u16::try_from(*global_heap_index)
         .map_err(|_| vds_err("VDS mapping heap index out of range"))?;
     let obj = coll
@@ -329,7 +334,11 @@ enum Step {
 /// Work out the extent libhdf5 gives the virtual dataset
 /// (`H5D__virtual_set_extent_unlim`, default view `H5D_VDS_LAST_AVAILABLE`
 /// with a printf gap of 0) and how much of each unlimited mapping is read.
-fn plan(mappings: &[Mapping], stored: &[u64], sources: &mut Sources) -> Result<Plan, FormatError> {
+fn plan<S: Storage + ?Sized>(
+    mappings: &[Mapping],
+    stored: &[u64],
+    sources: &mut Sources<'_, '_, S>,
+) -> Result<Plan, FormatError> {
     let overflow = || FormatError::Overflow("VDS extent overflow".into());
     let rank = stored.len();
     let mut new_dims: Vec<Option<u64>> = vec![None; rank];
@@ -471,6 +480,25 @@ pub fn virtual_dataset_extent(
     _offset_size: u8,
     length_size: u8,
     resolver: Option<&VdsFileResolver>,
+) -> Result<Vec<u64>, FormatError> {
+    virtual_dataset_extent_in(
+        file_data,
+        layout,
+        dataspace,
+        _offset_size,
+        length_size,
+        resolver,
+    )
+}
+
+/// [`virtual_dataset_extent`] over any [`Storage`].
+pub fn virtual_dataset_extent_in<S: Storage + ?Sized>(
+    file_data: &S,
+    layout: &DataLayout,
+    dataspace: &Dataspace,
+    _offset_size: u8,
+    length_size: u8,
+    resolver: Option<&VdsFileResolver>,
 ) -> Result<Vec<u64>, FormatError> {
     let mappings = load_mappings(file_data, layout, length_size)?;
     if mappings.iter().all(|m| m.kind == Kind::Fixed) {
@@ -497,6 +525,30 @@ pub fn read_virtual_dataset(
     _offset_size: u8,
     length_size: u8,
     resolver: Option<&VdsFileResolver>,
+) -> Result<VirtualData, FormatError> {
+    read_virtual_dataset_in(
+        file_data,
+        layout,
+        dataspace,
+        datatype,
+        fill,
+        _offset_size,
+        length_size,
+        resolver,
+    )
+}
+
+/// [`read_virtual_dataset`] over any [`Storage`].
+#[allow(clippy::too_many_arguments)]
+pub fn read_virtual_dataset_in<S: Storage + ?Sized>(
+    file_data: &S,
+    layout: &DataLayout,
+    dataspace: &Dataspace,
+    datatype: &Datatype,
+    fill: Option<&[u8]>,
+    _offset_size: u8,
+    length_size: u8,
+    resolver: Option<&VdsFileResolver>,
 ) -> Result<VirtualData, FormatError> {
     let mappings = load_mappings(file_data, layout, length_size)?;
     let mut sources = Sources::new(file_data, resolver);
@@ -510,7 +562,7 @@ pub fn read_virtual_dataset(
     let mut data = crate::chunked_read::alloc_output(crate::chunked_read::checked_byte_len(
         total, elem_size,
     )?)?;
-    if let Some(fill) = fill.filter(|f| f.len() == elem_size && f.iter().any(|&b| b != 0)) {
+    if let Some(fill) = fill.filter(|f| <[u8]>::len(f) == elem_size && f.iter().any(|&b| b != 0)) {
         for element in data.chunks_exact_mut(elem_size) {
             element.copy_from_slice(fill);
         }
@@ -611,12 +663,12 @@ fn scatter(
         return Err(vds_err("virtual/source selection element counts differ"));
     }
     for (&v, &s) in vidx.iter().zip(sidx) {
-        let (vo, so) = (v as usize * elem_size, s as usize * elem_size);
+        let (vo, so) = (to_usize(v)? * elem_size, to_usize(s)? * elem_size);
         if vo + elem_size > out.len() || so + elem_size > src.len() {
             return Err(vds_err("virtual dataset selection out of bounds"));
         }
         out[vo..vo + elem_size].copy_from_slice(&src[so..so + elem_size]);
-        mapped[v as usize] = true;
+        mapped[to_usize(v)?] = true;
     }
     Ok(())
 }
@@ -747,7 +799,7 @@ fn selection_indices(
                     return Err(vds_err("VDS selection blocks overlap"));
                 }
             }
-            let mut out = Vec::with_capacity(volume as usize);
+            let mut out = Vec::with_capacity(to_usize(volume)?);
             for (s, e) in starts.chunks_exact(rank).zip(ends.chunks_exact(rank)) {
                 let mut cur = s.to_vec();
                 'block: loop {
@@ -779,14 +831,17 @@ struct SourceData {
 
 /// Source files and datasets, fetched on demand. The most recently used
 /// external file is kept, since consecutive mappings usually share one.
-struct Sources<'a, 'r> {
+///
-    file_data: &'a [u8],
+/// The virtual dataset's own file (`"."`) is read through its [`Storage`];
+/// an external source file is loaded whole, through the resolver.
+struct Sources<'a, 'r, S: Storage + ?Sized> {
+    file_data: &'a S,
     resolver: Option<&'r VdsFileResolver<'r>>,
     cached_file: Option<(String, Option<Vec<u8>>)>,
 }
 
-impl<'a, 'r> Sources<'a, 'r> {
+impl<'a, 'r, S: Storage + ?Sized> Sources<'a, 'r, S> {
-    fn new(file_data: &'a [u8], resolver: Option<&'r VdsFileResolver<'r>>) -> Self {
+    fn new(file_data: &'a S, resolver: Option<&'r VdsFileResolver<'r>>) -> Self {
         Sources {
             file_data,
             resolver,
@@ -794,16 +849,18 @@ impl<'a, 'r> Sources<'a, 'r> {
         }
     }
 
-    /// The bytes of source file `name`, or `None` if it does not exist.
+    /// The bytes of external source file `name` (not `"."`), or `None` if it
-    fn file(&mut self, name: &str) -> Result<Option<&[u8]>, FormatError> {
+    /// does not exist.
-        if name == "." {
+    fn external(&mut self, name: &str) -> Result<Option<&[u8]>, FormatError> {
-            return Ok(Some(self.file_data));
-        }
         if self.cached_file.as_ref().is_none_or(|(n, _)| n != name) {
             let resolver = self.resolver.ok_or_else(|| {
                 vds_err("external-file virtual dataset sources require a file resolver")
             })?;
-            self.cached_file = Some((String::from(name), resolver(name)?));
+            let mut bytes = resolver(name)?;
+            if let Some(b) = bytes.as_mut() {
+                load_source_file(b)?;
+            }
+            self.cached_file = Some((String::from(name), bytes));
         }
         // An external file is handed over whole; its addresses are relative
         // to its superblock, so skip any user block.
@@ -816,7 +873,10 @@ impl<'a, 'r> Sources<'a, 'r> {
     /// The extent of source dataset `path` in file `file`, or `None` when
     /// either does not exist.
     fn dims(&mut self, file: &str, path: &str) -> Result<Option<Vec<u64>>, FormatError> {
-        let Some(bytes) = self.file(file)? else {
+        if file == "." {
+            return Ok(open_source(self.file_data, path)?.map(|s| s.dataspace.dimensions));
+        }
+        let Some(bytes) = self.external(file)? else {
             return Ok(None);
         };
         Ok(open_source(bytes, path)?.map(|s| s.dataspace.dimensions))
@@ -841,7 +901,13 @@ impl<'a, 'r> Sources<'a, 'r> {
                  from another file is not supported"
             )));
         }
-        let Some(bytes) = self.file(file)? else {
+        if file == "." {
+            let Some(src) = open_source(self.file_data, path)? else {
+                return Ok(None);
+            };
+            return read_source(self.file_data, src, path, datatype).map(Some);
+        }
+        let Some(bytes) = self.external(file)? else {
             return Ok(None);
         };
         let Some(src) = open_source(bytes, path)? else {
@@ -851,6 +917,23 @@ impl<'a, 'r> Sources<'a, 'r> {
     }
 }
 
+/// Check an external source file's superblock extension as libhdf5 does
+/// when it opens the file, and write any metadata cache image over its
+/// metadata in place: libhdf5 reads the image's entries instead of the
+/// file's own, possibly stale, bytes (`crate::superblock_ext`). A source
+/// file whose image cannot be loaded is an error, as other corrupt source
+/// files are here.
+fn load_source_file(whole: &mut [u8]) -> Result<(), FormatError> {
+    let base = crate::signature::find_signature(whole)?;
+    let sb = crate::superblock::Superblock::parse(&whole[base..], 0)?;
+    // The end of file the superblock records; a truncated source file is
+    // read as before, up to its length.
+    let end = sb
+        .data_end(base as u64, whole.len() as u64)
+        .map_or(Ok(whole.len()), |e| to_usize(e).map(|e| base + e))?;
+    crate::superblock_ext::apply_cache_image_in_place(&mut whole[base..end], &sb)
+}
+
 /// Whether elements of `dt` contain addresses into their own file:
 /// variable-length data (global-heap IDs) or references.
 fn holds_file_addresses(dt: &Datatype) -> bool {
@@ -888,19 +971,23 @@ fn source_message<'h>(
 
 /// Open source dataset `path` of the file in `file_data`, or `None` if there
 /// is no such object (libhdf5 reads a missing source as fill).
-fn open_source(file_data: &[u8], path: &str) -> Result<Option<OpenSource>, FormatError> {
+fn open_source<S: Storage + ?Sized>(
+    file_data: &S,
+    path: &str,
+) -> Result<Option<OpenSource>, FormatError> {
     use crate::message_type::MessageType;
-    use crate::shared_message::message_data_with_sohm;
+    use crate::shared_message::message_data_with_sohm_in as message_data_with_sohm;
 
-    // `file_data` starts at the superblock (see `Sources::file`).
+    // `file_data` starts at the superblock (see `Sources::external`).
-    let sb = crate::superblock::Superblock::parse(file_data, 0)?;
+    let sb = crate::superblock::Superblock::parse_in(file_data, 0)?;
     let (os, ls) = (sb.offset_size, sb.length_size);
-    let addr = match crate::group_v2::resolve_path_any(file_data, &sb, path) {
+    let addr = match crate::group_v2::resolve_path_any_in(file_data, &sb, path) {
         Ok(a) => a,
         Err(FormatError::PathNotFound(_)) => return Ok(None),
         Err(e) => return Err(e),
     };
-    let header = crate::object_header::ObjectHeader::parse(file_data, addr as usize, os, ls)?;
+    let header =
+        crate::object_header::ObjectHeader::parse_in(file_data, checked_addr(addr)?, os, ls)?;
     let mut src = OpenSource {
         offset_size: os,
         length_size: ls,
@@ -919,15 +1006,15 @@ fn open_source(file_data: &[u8], path: &str) -> Result<Option<OpenSource>, Forma
 
 /// Read an opened source dataset in full (its own fill value applied to
 /// unallocated chunks).
-fn read_source(
+fn read_source<S: Storage + ?Sized>(
-    file_data: &[u8],
+    file_data: &S,
     src: OpenSource,
     path: &str,
     datatype: &Datatype,
 ) -> Result<SourceData, FormatError> {
     use crate::filter_pipeline::FilterPipeline;
     use crate::message_type::MessageType;
-    use crate::shared_message::message_data_with_sohm;
+    use crate::shared_message::message_data_with_sohm_in as message_data_with_sohm;
 
     let (os, ls) = (src.offset_size, src.length_size);
     let dt_msg = source_message(&src, path, MessageType::Datatype)?;
@@ -965,7 +1052,7 @@ fn read_source(
             message_data_with_sohm(file_data, m, os, ls).and_then(|d| FilterPipeline::parse(&d))
         })
         .transpose()?;
-    let raw = crate::fill_value::read_full_with_fill(
+    let raw = crate::fill_value::read_full_with_fill_in(
         &src.header.messages,
         file_data,
         &layout,
@@ -974,7 +1061,7 @@ fn read_source(
         os,
         ls,
         || {
-            crate::data_read::read_raw_data_full(
+            crate::data_read::read_raw_data_full_in(
                 file_data,
                 &layout,
                 &src.dataspace,

crates/clawhdf5-format/src/vl_data.rs

@@ -5,10 +5,11 @@
 //! `sequence_length(4 LE) + collection_address(offset_size LE) + object_index(4 LE)`.
 
 #[cfg(not(feature = "std"))]
-use alloc::{collections::BTreeMap, format, string::String, vec, vec::Vec};
+use alloc::{borrow::Cow, collections::BTreeMap, format, string::String, vec, vec::Vec};
 #[cfg(feature = "std")]
-use std::collections::BTreeMap;
+use std::{borrow::Cow, collections::BTreeMap};
 
+use crate::addr::to_usize;
 use crate::error::FormatError;
 use crate::global_heap::{GlobalHeapCollection, GlobalHeapIndex};
 
@@ -55,7 +56,7 @@ pub fn parse_vl_references(
 ) -> Result<Vec<VlElement>, FormatError> {
     let elem_size = 4 + offset_size as usize + 4; // length + address + index
     let total =
-        (num_elements as usize)
+        to_usize(num_elements)?
             .checked_mul(elem_size)
             .ok_or(FormatError::UnexpectedEof {
                 expected: usize::MAX,
@@ -68,7 +69,7 @@ pub fn parse_vl_references(
         });
     }
 
-    let mut elements = Vec::with_capacity(num_elements as usize);
+    let mut elements = Vec::with_capacity(to_usize(num_elements)?);
     let mut pos = 0;
 
     for _ in 0..num_elements {
@@ -136,13 +137,15 @@ pub fn check_element_size(stored_size: u32, offset_size: u8) -> Result<(), Forma
 
 /// A collection's objects, located in the file data but not copied:
 /// `(index, offset, size)` of the first object with each index, sorted by
-/// index.
+/// index. Over a storage without the whole file in memory, also the
-struct CachedCollection {
+/// collection's bytes (`(offset, bytes)`), read once when it is indexed.
+struct CachedCollection<'a> {
     objects: Vec<(u16, usize, usize)>,
+    bytes: Option<(usize, Cow<'a, [u8]>)>,
 }
 
-impl CachedCollection {
+impl<'a> CachedCollection<'a> {
-    fn new(index: GlobalHeapIndex) -> Self {
+    fn new(index: GlobalHeapIndex, bytes: Option<(usize, Cow<'a, [u8]>)>) -> Self {
         let mut objects: Vec<(u16, usize, usize)> = index
             .objects
             .iter()
@@ -151,12 +154,16 @@ impl CachedCollection {
         // Stable, so the first object with a repeated index is kept.
         objects.sort_by_key(|o| o.0);
         objects.dedup_by_key(|o| o.0);
-        Self { objects }
+        Self { objects, bytes }
     }
 
     /// What this entry costs to keep, in bytes (roughly).
     fn cost(&self) -> usize {
-        64 + self.objects.len() * core::mem::size_of::<(u16, usize, usize)>()
+        let held = match &self.bytes {
+            Some((_, Cow::Owned(b))) => b.len(),
+            _ => 0,
+        };
+        64 + self.objects.len() * core::mem::size_of::<(u16, usize, usize)>() + held
     }
 
     fn get(&self, index: u32) -> Option<(usize, usize)> {
@@ -169,6 +176,8 @@ impl CachedCollection {
 /// How many bytes of collection indexes a [`VlResolver`] keeps before it
 /// drops them and starts again. Values are never copied into the cache, so
 /// this bounds what a read retains however many collections it visits.
+/// (Over a storage without the whole file in memory the collections' bytes
+/// are kept too, and count against this.)
 const CACHE_BUDGET: usize = 32 << 20;
 
 /// Resolves variable-length elements against a file's global heap, parsing
@@ -184,11 +193,18 @@ const CACHE_BUDGET: usize = 32 << 20;
 /// that overlap one another are refused (libhdf5 never writes them), so a
 /// file cannot make the resolver parse the same bytes as the objects of
 /// many collections.
-pub struct VlResolver<'a> {
+///
-    file_data: &'a [u8],
+/// The file is any [`Storage`](crate::storage::Storage) (`S`, a slice by default). Over one without
+/// the whole file in memory each collection is read once, when first used,
+/// and kept (within the budget above); [`Self::strings`],
+/// [`Self::string_bytes`] and [`Self::sequences`] work over any storage,
+/// [`Self::element`] and [`Self::string_element`], which borrow from the
+/// file, over a slice.
+pub struct VlResolver<'a, S: crate::storage::Storage + ?Sized = [u8]> {
+    file_data: &'a S,
     offset_size: u8,
     length_size: u8,
-    cache: BTreeMap<u64, CachedCollection>,
+    cache: BTreeMap<u64, CachedCollection<'a>>,
     cached_bytes: usize,
     budget: usize,
     /// Start → end of every collection parsed so far (kept when the cache
@@ -200,6 +216,56 @@ impl<'a> VlResolver<'a> {
     /// A resolver over `file_data` (the file from its superblock on), with
     /// the superblock's offset and length sizes.
     pub fn new(file_data: &'a [u8], offset_size: u8, length_size: u8) -> Self {
+        Self::new_in(file_data, offset_size, length_size)
+    }
+
+    /// One element (the first [`element_size`](Self::element_size) bytes of
+    /// `elem`) of a variable-length sequence whose base type is `base_size`
+    /// bytes: its `length × base_size` bytes, or `None` for a null element
+    /// (heap address 0).
+    pub fn element(
+        &mut self,
+        elem: &[u8],
+        base_size: usize,
+    ) -> Result<Option<&'a [u8]>, FormatError> {
+        let vl = parse_vl_references(elem, 1, self.offset_size)?;
+        let vl = &vl[0];
+        if vl.collection_address == 0 {
+            return Ok(None);
+        }
+        let (start, size) = self.locate(vl)?;
+        let data = &self.file_data[start..start + size];
+        check_object_size(vl, data.len(), base_size)?;
+        Ok(Some(data))
+    }
+
+    /// One variable-length string element: its bytes up to the first NUL,
+    /// or `None` for a null element (h5dump prints it as `NULL`, h5py
+    /// returns it as empty).
+    pub fn string_element(&mut self, elem: &[u8]) -> Result<Option<&'a [u8]>, FormatError> {
+        Ok(self.element(elem, 1)?.map(cut_at_nul))
+    }
+}
+
+/// `data_len`, the size of `vl`'s heap object, against the `length ×
+/// base_size` bytes the element says it holds.
+fn check_object_size(vl: &VlElement, data_len: usize, base_size: usize) -> Result<(), FormatError> {
+    let expected = (vl.length as usize)
+        .checked_mul(base_size)
+        .ok_or_else(|| FormatError::Overflow("variable-length element size".into()))?;
+    if data_len != expected {
+        return Err(FormatError::VlDataError(format!(
+            "global heap object {} in the collection at {} holds {data_len} bytes; the element \
+             says {} × {base_size}",
+            vl.object_index, vl.collection_address, vl.length
+        )));
+    }
+    Ok(())
+}
+
+impl<'a, S: crate::storage::Storage + ?Sized> VlResolver<'a, S> {
+    /// [`VlResolver::new`] over any [`Storage`](crate::storage::Storage).
+    pub fn new_in(file_data: &'a S, offset_size: u8, length_size: u8) -> Self {
         Self {
             file_data,
             offset_size,
@@ -230,49 +296,31 @@ impl<'a> VlResolver<'a> {
 
     /// The bytes of one element: `length × base_size` bytes from the heap,
     /// or `None` for a null element.
-    fn resolve(
+    fn resolve(&mut self, vl: &VlElement, base_size: usize) -> Result<Option<&[u8]>, FormatError> {
-        &mut self,
+        if vl.collection_address == 0 {
-        vl: &VlElement,
-        base_size: usize,
-    ) -> Result<Option<&'a [u8]>, FormatError> {
-        let addr = vl.collection_address;
-        if addr == 0 {
             return Ok(None);
         }
         let data = self.object(vl)?;
-        let expected = (vl.length as usize)
+        check_object_size(vl, data.len(), base_size)?;
-            .checked_mul(base_size)
-            .ok_or_else(|| FormatError::Overflow("variable-length element size".into()))?;
-        if data.len() != expected {
-            return Err(FormatError::VlDataError(format!(
-                "global heap object {} in the collection at {addr} holds {} bytes; the element \
-                 says {} × {base_size}",
-                vl.object_index,
-                data.len(),
-                vl.length
-            )));
-        }
         Ok(Some(data))
     }
 
-    /// One element (the first [`element_size`](Self::element_size) bytes of
+    /// [`VlResolver::element`] over any storage: the element's bytes
-    /// `elem`) of a variable-length sequence whose base type is `base_size`
+    /// (borrowed from the resolver's cache of heap collections, so they
-    /// bytes: its `length × base_size` bytes, or `None` for a null element
+    /// live until the next call), or `None` for a null element.
-    /// (heap address 0).
+    pub fn element_in(
-    pub fn element(
         &mut self,
         elem: &[u8],
         base_size: usize,
-    ) -> Result<Option<&'a [u8]>, FormatError> {
+    ) -> Result<Option<&[u8]>, FormatError> {
         let vl = parse_vl_references(elem, 1, self.offset_size)?;
         self.resolve(&vl[0], base_size)
     }
 
-    /// One variable-length string element: its bytes up to the first NUL,
+    /// [`VlResolver::string_element`] over any storage (see
-    /// or `None` for a null element (h5dump prints it as `NULL`, h5py
+    /// [`element_in`](Self::element_in)).
-    /// returns it as empty).
+    pub fn string_element_in(&mut self, elem: &[u8]) -> Result<Option<&[u8]>, FormatError> {
-    pub fn string_element(&mut self, elem: &[u8]) -> Result<Option<&'a [u8]>, FormatError> {
+        Ok(self.element_in(elem, 1)?.map(cut_at_nul))
-        Ok(self.element(elem, 1)?.map(cut_at_nul))
     }
 
     /// The strings of the variable-length string elements in `raw`, as
@@ -329,9 +377,20 @@ pub fn read_vl_strings(
     num_elements: u64,
     offset_size: u8,
     length_size: u8,
+) -> Result<Vec<String>, FormatError> {
+    read_vl_strings_in(file_data, raw_data, num_elements, offset_size, length_size)
+}
+
+/// [`read_vl_strings`] over any [`Storage`](crate::storage::Storage).
+pub fn read_vl_strings_in<S: crate::storage::Storage + ?Sized>(
+    file_data: &S,
+    raw_data: &[u8],
+    num_elements: u64,
+    offset_size: u8,
+    length_size: u8,
 ) -> Result<Vec<String>, FormatError> {
     let raw = first_elements(raw_data, num_elements, offset_size)?;
-    VlResolver::new(file_data, offset_size, length_size).strings(raw)
+    VlResolver::new_in(file_data, offset_size, length_size).strings(raw)
 }
 
 /// The first `num_elements` elements of `raw`, or an error if it is shorter.
@@ -363,9 +422,20 @@ pub fn read_vl_bytes(
     num_elements: u64,
     offset_size: u8,
     length_size: u8,
+) -> Result<Vec<Vec<u8>>, FormatError> {
+    read_vl_bytes_in(file_data, raw_data, num_elements, offset_size, length_size)
+}
+
+/// [`read_vl_bytes`] over any [`Storage`](crate::storage::Storage).
+pub fn read_vl_bytes_in<S: crate::storage::Storage + ?Sized>(
+    file_data: &S,
+    raw_data: &[u8],
+    num_elements: u64,
+    offset_size: u8,
+    length_size: u8,
 ) -> Result<Vec<Vec<u8>>, FormatError> {
     let refs = parse_vl_references(raw_data, num_elements, offset_size)?;
-    let mut resolver = VlResolver::new(file_data, offset_size, length_size);
+    let mut resolver = VlResolver::new_in(file_data, offset_size, length_size);
     let mut result = Vec::with_capacity(refs.len());
 
     for vl in &refs {
@@ -384,10 +454,10 @@ pub fn read_vl_bytes(
     Ok(result)
 }
 
-impl<'a> VlResolver<'a> {
+impl<'a, S: crate::storage::Storage + ?Sized> VlResolver<'a, S> {
-    /// The heap object `vl` points to, whatever its size; its collection is
+    /// Where the heap object `vl` points to lies in the file, whatever its
-    /// parsed on first use.
+    /// size (`(offset, size)`); its collection is parsed on first use.
-    fn object(&mut self, vl: &VlElement) -> Result<&'a [u8], FormatError> {
+    fn locate(&mut self, vl: &VlElement) -> Result<(usize, usize), FormatError> {
         let addr = vl.collection_address;
         // libhdf5 writes a null element with address 0, never the undefined
         // address, and fails to read one ("addr undefined") even when its
@@ -401,14 +471,20 @@ impl<'a> VlResolver<'a> {
         if !self.cache.contains_key(&addr) {
             let offset = usize::try_from(addr).map_err(|_| FormatError::UnexpectedEof {
                 expected: usize::MAX,
-                available: self.file_data.len(),
+                available: crate::storage::len_usize(self.file_data),
             })?;
-            let index =
+            let (bytes, base, index) =
-                GlobalHeapCollection::parse_index(self.file_data, offset, self.length_size)?;
+                GlobalHeapCollection::read_collection(self.file_data, addr, self.length_size)?;
-            // parse_index checked that the collection lies in the file.
+            // read_collection checked that the collection lies in the file.
-            let end = offset + index.collection_size as usize;
+            let end = offset + to_usize(index.collection_size)?;
             self.check_overlap(offset, end)?;
-            let coll = CachedCollection::new(index);
+            // With the whole file in memory the objects are sliced from it;
+            // otherwise the collection's bytes are kept.
+            let bytes = match self.file_data.as_contiguous() {
+                Some(_) => None,
+                None => Some((base, bytes)),
+            };
+            let coll = CachedCollection::new(index, bytes);
             if self.cached_bytes.saturating_add(coll.cost()) > self.budget {
                 self.cache.clear();
                 self.cached_bytes = 0;
@@ -416,13 +492,27 @@ impl<'a> VlResolver<'a> {
             self.cached_bytes += coll.cost();
             self.cache.insert(addr, coll);
         }
-        let (start, size) = self.cache[&addr].get(vl.object_index).ok_or(
+        self.cache[&addr]
-            FormatError::GlobalHeapObjectNotFound {
+            .get(vl.object_index)
+            .ok_or(FormatError::GlobalHeapObjectNotFound {
                 collection_address: addr,
                 index: vl.object_index as u16,
-            },
+            })
-        )?;
+    }
-        Ok(&self.file_data[start..start + size])
+
+    /// The heap object `vl` points to, whatever its size; its collection is
+    /// parsed on first use.
+    fn object(&mut self, vl: &VlElement) -> Result<&[u8], FormatError> {
+        let (start, size) = self.locate(vl)?;
+        if let Some(all) = self.file_data.as_contiguous() {
+            return Ok(&all[start..start + size]);
+        }
+        match &self.cache[&vl.collection_address].bytes {
+            Some((base, bytes)) => Ok(&bytes[start - base..start - base + size]),
+            None => Err(FormatError::Storage(
+                "global heap collection bytes were not kept".into(),
+            )),
+        }
     }
 
     /// Record the collection at `start..end`, refusing one that overlaps a
@@ -573,6 +663,32 @@ mod tests {
         }
     }
 
+    #[test]
+    fn element_in_over_a_storage_matches_element_over_a_slice() {
+        let mut file_data = vec![0u8; 512];
+        build_gcol_at(&mut file_data, 256, &[(1, b"Alice\0x"), (2, b"Bob")]);
+        let mut raw = build_vl_refs(&["Alice\0x", "Bob"], 256, 1, 8);
+        raw.extend(element(0, 0, 0, 8)); // null
+        raw.extend(element(9, 256, 1, 8)); // wrong length: an error
+        let storage = crate::storage::CountingStorage::new(file_data.clone());
+        let dynamic: &dyn crate::storage::Storage = &storage;
+        let mut slice = VlResolver::new(&file_data, 8, 8);
+        let mut any = VlResolver::new_in(dynamic, 8, 8);
+        for e in raw.chunks(16) {
+            let want = slice.element(e, 1).map(|o| o.map(<[u8]>::to_vec));
+            let got = any.element_in(e, 1).map(|o| o.map(<[u8]>::to_vec));
+            assert_eq!(format!("{want:?}"), format!("{got:?}"));
+            let want = slice.string_element(e).map(|o| o.map(<[u8]>::to_vec));
+            let got = any.string_element_in(e).map(|o| o.map(<[u8]>::to_vec));
+            assert_eq!(format!("{want:?}"), format!("{got:?}"));
+        }
+        assert_eq!(
+            any.string_element_in(&raw[..16]).unwrap(),
+            Some(&b"Alice"[..])
+        );
+        assert!(storage.reads() > 0);
+    }
+
     #[test]
     fn null_vl_element_zero_address() {
         let mut raw = Vec::new();
@@ -708,6 +824,7 @@ mod tests {
         let mut r = VlResolver::new(&file_data, 8, 8);
         let one = CachedCollection {
             objects: vec![(0, 0, 0); 3],
+            bytes: None,
         }
         .cost();
         r.budget = 2 * one + 1;

crates/clawhdf5-format/tests/blosc2_alloc_bounds.rs

@@ -0,0 +1,641 @@
+//! Crafted Blosc2 frames and chunks cannot make the decoder allocate out of
+//! proportion to the HDF5 chunk it decodes.
+//!
+//! A frame's header, its offsets chunk and its chunk headers all declare
+//! sizes, and the decoder used to allocate what they declared: a 173-byte
+//! frame whose offsets chunk claimed 2 GiB was decoded in full before any
+//! check failed. Every allocation is now bounded by the output limit (the
+//! HDF5 chunk's size) and the input's length.
+//!
+//! Peak heap use is measured with a counting global allocator; the tests
+//! share it, so each holds `SERIAL` for its whole run.
+#![cfg(feature = "blosc2")]
+
+use std::alloc::{GlobalAlloc, Layout, System};
+use std::sync::Mutex;
+use std::sync::atomic::{AtomicUsize, Ordering};
+
+use clawhdf5_format::filters_blosc2::{blosc2_decompress, blosc2_decompress_chunk};
+
+struct Counting;
+
+static CURRENT: AtomicUsize = AtomicUsize::new(0);
+static PEAK: AtomicUsize = AtomicUsize::new(0);
+static SERIAL: Mutex<()> = Mutex::new(());
+
+unsafe impl GlobalAlloc for Counting {
+    unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
+        let p = unsafe { System.alloc(layout) };
+        if !p.is_null() {
+            let now = CURRENT.fetch_add(layout.size(), Ordering::Relaxed) + layout.size();
+            PEAK.fetch_max(now, Ordering::Relaxed);
+        }
+        p
+    }
+
+    unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 {
+        let p = unsafe { System.alloc_zeroed(layout) };
+        if !p.is_null() {
+            let now = CURRENT.fetch_add(layout.size(), Ordering::Relaxed) + layout.size();
+            PEAK.fetch_max(now, Ordering::Relaxed);
+        }
+        p
+    }
+
+    unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {
+        unsafe { System.dealloc(ptr, layout) };
+        CURRENT.fetch_sub(layout.size(), Ordering::Relaxed);
+    }
+}
+
+#[global_allocator]
+static ALLOC: Counting = Counting;
+
+/// Bytes allocated at the peak of `f`, above what was live when it started.
+fn peak_during<T>(f: impl FnOnce() -> T) -> (T, usize) {
+    let base = CURRENT.load(Ordering::Relaxed);
+    PEAK.store(base, Ordering::Relaxed);
+    let out = f();
+    (out, PEAK.load(Ordering::Relaxed).saturating_sub(base))
+}
+
+/// What decoding one HDF5 chunk of `limit` bytes from `input` may hold at
+/// once: the output, a few blocks of scratch (each no larger than the
+/// output), the offsets table, and the Zstandard decoder's state, which has
+/// a fixed ceiling: a window of at most 128 KiB (or twice the stream) and a
+/// block's table of sequences (up to 98,303 of 12 bytes, 1.2 MB).
+fn bound(limit: usize, input: &[u8]) -> usize {
+    6 * limit + 2 * input.len() + (2 << 20)
+}
+
+fn lock() -> std::sync::MutexGuard<'static, ()> {
+    SERIAL.lock().unwrap_or_else(|e| e.into_inner())
+}
+
+/// A 32-byte (extended) Blosc2 chunk header.
+fn chunk_header(ts: u8, nbytes: i32, blocksize: i32, cbytes: i32, special: u8) -> Vec<u8> {
+    let mut c = vec![5u8, 1, 0x05, ts];
+    for v in [nbytes, blocksize, cbytes] {
+        c.extend_from_slice(&v.to_le_bytes());
+    }
+    c.resize(32, 0);
+    c[31] = special << 4;
+    c
+}
+
+/// A chunk of `nbytes` bytes that repeats one value (special type 3).
+fn repeated(value: &[u8], nbytes: i32, blocksize: i32) -> Vec<u8> {
+    let mut c = chunk_header(
+        value.len() as u8,
+        nbytes,
+        blocksize,
+        32 + value.len() as i32,
+        3,
+    );
+    c.extend_from_slice(value);
+    c
+}
+
+/// A frame offset recording a special chunk of `kind` (1 zeros, 2 NaN).
+fn special_offset(kind: u8) -> [u8; 8] {
+    (((0x80 | kind) as i64) << 56).to_le_bytes()
+}
+
+/// A B2ND metalayer.
+fn nd_meta(shape: &[i64], chunks: &[i32], blocks: &[i32]) -> Vec<u8> {
+    let n = shape.len() as u8;
+    let mut m = vec![0x95, 0, n, 0x90 | n];
+    for s in shape {
+        m.push(0xd3);
+        m.extend_from_slice(&s.to_be_bytes());
+    }
+    for dims in [chunks, blocks] {
+        m.push(0x90 | n);
+        for d in dims {
+            m.push(0xd2);
+            m.extend_from_slice(&d.to_be_bytes());
+        }
+    }
+    m
+}
+
+/// A contiguous frame: header (with a `b2nd` metalayer if given), the data
+/// chunks, then the offsets chunk.
+fn frame(
+    meta: Option<&[u8]>,
+    nbytes: i64,
+    typesize: i32,
+    chunksize: i32,
+    data: &[u8],
+    offsets: &[u8],
+) -> Vec<u8> {
+    let mut h = vec![0u8; 91];
+    h[0] = 0x9e;
+    h[1] = 0xa8;
+    h[2..10].copy_from_slice(b"b2frame\0");
+    h[25] = 2;
+    match meta {
+        Some(m) => {
+            h.extend_from_slice(&[0xde, 0, 1, 0xa4]);
+            h.extend_from_slice(b"b2nd");
+            let at = h.len() as i32 + 5;
+            h.push(0xd2);
+            h.extend_from_slice(&at.to_be_bytes());
+            h.push(0xc6);
+            h.extend_from_slice(&(m.len() as u32).to_be_bytes());
+            h.extend_from_slice(m);
+        }
+        None => h.extend_from_slice(&[0xde, 0, 0]),
+    }
+    let header_len = h.len() as i32;
+    h[11..15].copy_from_slice(&header_len.to_be_bytes());
+    h[30..38].copy_from_slice(&nbytes.to_be_bytes());
+    h[39..47].copy_from_slice(&(data.len() as i64).to_be_bytes());
+    h[48..52].copy_from_slice(&typesize.to_be_bytes());
+    h[58..62].copy_from_slice(&chunksize.to_be_bytes());
+    h.extend_from_slice(data);
+    h.extend_from_slice(offsets);
+    let len = h.len() as u64;
+    h[16..24].copy_from_slice(&len.to_be_bytes());
+    h
+}
+
+/// The frame header's own sizes must not size the offsets chunk: a frame
+/// declaring 32 Mi chunks of 4 bytes, whose offsets chunk (40 bytes) says
+/// "one repeated offset, 256 MiB of them", made the decoder build all
+/// 256 MiB of offsets for a 1 MiB HDF5 chunk and then return 4 bytes.
+#[test]
+fn offsets_chunk_is_bounded_by_the_output_limit() {
+    let _g = lock();
+    let limit = 1 << 20;
+    let offsets_len: i32 = 256 << 20;
+    let nchunks = offsets_len as i64 / 8;
+    let offsets = repeated(&special_offset(1), offsets_len, 64 << 20);
+    let f = frame(None, nchunks * 4, 4, 4, &[], &offsets);
+    let (r, peak) = peak_during(|| blosc2_decompress(&f, limit));
+    assert!(r.is_err(), "decoded {:?} bytes", r.map(|v| v.len()));
+    assert!(
+        peak <= bound(limit, &f),
+        "peak {peak} bytes for a {}-byte frame",
+        f.len()
+    );
+    // The same frame with a variable chunk size (0): the offsets chunk
+    // alone says how many chunks there are.
+    let f = frame(None, nchunks * 4, 4, 0, &[], &offsets);
+    let (r, peak) = peak_during(|| blosc2_decompress(&f, limit));
+    assert!(r.is_err());
+    assert!(peak <= bound(limit, &f), "chunksize 0: peak {peak} bytes");
+}
+
+/// A legitimate frame of this shape (one chunk, its offset special) still
+/// decodes.
+#[test]
+fn small_frames_still_decode() {
+    let _g = lock();
+    let offsets = repeated(&special_offset(1), 8, 8);
+    let f = frame(None, 64, 4, 64, &[], &offsets);
+    assert_eq!(blosc2_decompress(&f, 64).unwrap(), vec![0; 64]);
+    let _ = blosc2_decompress_chunk;
+}
+
+/// A chunk that decodes to nothing kept its declared block size (up to
+/// 512 MiB) and allocated two scratch blocks of it: about 1 GiB for a
+/// 20-byte chunk.
+#[test]
+fn empty_chunk_does_not_allocate_its_block_size() {
+    let _g = lock();
+    let mut c = vec![5u8, 1, 0x01, 1];
+    for v in [0i32, 0x1FFF_F000, 20] {
+        c.extend_from_slice(&v.to_le_bytes());
+    }
+    c.resize(20, 0);
+    let (r, peak) = peak_during(|| blosc2_decompress_chunk(&c, 1 << 20));
+    assert_eq!(r.map(|v| v.len()).unwrap_or(0), 0);
+    assert!(
+        peak <= bound(0, &c),
+        "peak {peak} bytes for a 20-byte chunk"
+    );
+    // Inside a frame for a non-empty HDF5 chunk it is an error, not data.
+    let offsets = repeated(&0i64.to_le_bytes(), 8, 8);
+    let f = frame(None, 64, 4, 64, &c, &offsets);
+    let (r, peak) = peak_during(|| blosc2_decompress(&f, 64));
+    assert!(r.is_err(), "decoded {:?}", r.map(|v| v.len()));
+    assert!(peak <= bound(64, &f), "in a frame: peak {peak} bytes");
+}
+
+/// B2ND chunks were decoded whole, padding included, with up to 16x the
+/// HDF5 chunk size as their limit. Blocks are now placed as they are
+/// decoded, so the padding is never held.
+///
+/// Ten dimensions: nine of 3 split into blocks of 2 (padded to 4) and one
+/// of 4, so each chunk is 13x the array. One chunk, stored three ways: as a
+/// NaN chunk in the frame's offsets, as a repeated-value chunk, and as a
+/// chunk of stored (uncompressed) blocks.
+#[test]
+fn b2nd_padding_is_never_held() {
+    let _g = lock();
+    let ts = 4usize;
+    let mut shape = vec![3i64; 9];
+    shape.push(4);
+    let chunks: Vec<i32> = shape.iter().map(|&s| s as i32).collect();
+    let mut blocks = vec![2i32; 9];
+    blocks.push(4);
+    let meta = nd_meta(&shape, &chunks, &blocks);
+    let items: usize = shape.iter().product::<i64>() as usize;
+    let limit = items * ts;
+    let block_bytes = ts * blocks.iter().product::<i32>() as usize;
+    let ext_bytes = ts * 4usize.pow(9) * 4;
+    assert!(ext_bytes > 13 * limit);
+    let offsets = |off: [u8; 8]| repeated(&off, 8, 8);
+
+    let value = 1.5f32.to_le_bytes();
+    let stored = {
+        // Every block stored raw: block k holds the value k.
+        let mut c = chunk_header(4, ext_bytes as i32, block_bytes as i32, 0, 0);
+        c[2] = 0x02 | 0x10; // memcpyed, not split
+        c.truncate(16);
+        for k in 0..ext_bytes / block_bytes {
+            c.extend((k as f32).to_le_bytes().repeat(block_bytes / 4));
+        }
+        let n = c.len() as i32;
+        c[12..16].copy_from_slice(&n.to_le_bytes());
+        c
+    };
+    let cases: Vec<(&str, Vec<u8>)> = vec![
+        (
+            "NaN offset",
+            frame(
+                Some(&meta),
+                ext_bytes as i64,
+                4,
+                ext_bytes as i32,
+                &[],
+                &offsets(special_offset(2)),
+            ),
+        ),
+        (
+            "repeated value",
+            frame(
+                Some(&meta),
+                ext_bytes as i64,
+                4,
+                ext_bytes as i32,
+                &repeated(&value, ext_bytes as i32, block_bytes as i32),
+                &offsets(0i64.to_le_bytes()),
+            ),
+        ),
+        (
+            "stored blocks",
+            frame(
+                Some(&meta),
+                ext_bytes as i64,
+                4,
+                ext_bytes as i32,
+                &stored,
+                &offsets(0i64.to_le_bytes()),
+            ),
+        ),
+    ];
+    for (name, f) in cases {
+        let (r, peak) = peak_during(|| blosc2_decompress(&f, limit));
+        let out = r.unwrap_or_else(|e| panic!("{name}: {e}"));
+        assert_eq!(out.len(), limit, "{name}");
+        match name {
+            "NaN offset" => assert!(
+                out.chunks(4)
+                    .all(|v| f32::from_le_bytes(v.try_into().unwrap()).is_nan())
+            ),
+            "repeated value" => assert!(out.chunks(4).all(|v| v == value)),
+            _ => {
+                // Element (i0..i9) lies in block (i0/2, .., i8/2), numbered
+                // in C order over a 2x..x2x1 grid of blocks.
+                let mut idx = [0usize; 10];
+                for (e, v) in out.chunks(4).enumerate() {
+                    let mut n = e;
+                    for d in (0..10).rev() {
+                        idx[d] = n % shape[d] as usize;
+                        n /= shape[d] as usize;
+                    }
+                    let k = idx[..9].iter().fold(0, |k, &i| k * 2 + i / 2);
+                    assert_eq!(
+                        f32::from_le_bytes(v.try_into().unwrap()),
+                        k as f32,
+                        "{name} {e}"
+                    );
+                }
+            }
+        }
+        assert!(
+            peak <= bound(limit, &f),
+            "{name}: peak {peak} bytes for a {limit}-byte chunk ({}-byte frame)",
+            f.len()
+        );
+    }
+}
+
+/// A B2ND chunk larger than the array (here 16x, the old cap) is refused,
+/// or at least never allocated.
+#[test]
+fn b2nd_chunk_larger_than_the_array_is_not_allocated() {
+    let _g = lock();
+    let limit = 1 << 20;
+    let c = 16 * limit as i32;
+    let meta = nd_meta(&[limit as i64], &[c], &[c]);
+    let offsets = repeated(&special_offset(1), 8, 8);
+    let f = frame(Some(&meta), c as i64, 1, c, &[], &offsets);
+    let (r, peak) = peak_during(|| blosc2_decompress(&f, limit));
+    assert!(
+        peak <= bound(limit, &f),
+        "peak {peak} bytes ({:?})",
+        r.map(|v| v.len())
+    );
+}
+
+/// ruzstd reserves a frame's declared window (up to 100 MiB) before it
+/// decodes a frame with a decoder it has used before: a Blosc2 chunk of
+/// two 16-byte Zstandard streams, each declaring a 96 MiB window,
+/// allocated 96 MiB. c-blosc2 compresses each block with its size known,
+/// so its windows never exceed the block.
+#[test]
+fn zstd_window_is_bounded_by_the_output() {
+    let _g = lock();
+    let mut z = 0xfd2f_b528u32.to_le_bytes().to_vec();
+    // No single segment, no checksum; window 2^26 + 4/8 of it = 96 MiB.
+    z.extend_from_slice(&[0x00, (16 << 3) | 4]);
+    // One raw block, last, of 16 bytes.
+    let h = 1 | (16 << 3);
+    z.extend_from_slice(&[h as u8, (h >> 8) as u8, 0]);
+    z.extend_from_slice(&[7; 16]);
+    // Two blocks of 16 bytes, one stream each (not split), Zstandard
+    // (codec 4).
+    let chunk = |z: &[u8]| {
+        let mut c = vec![5u8, 1, 0x10 | (4 << 5), 1];
+        for v in [32i32, 16, 0] {
+            c.extend_from_slice(&v.to_le_bytes());
+        }
+        let first = 24 + 4 + z.len();
+        c.extend_from_slice(&24i32.to_le_bytes());
+        c.extend_from_slice(&(first as i32).to_le_bytes());
+        for _ in 0..2 {
+            c.extend_from_slice(&(z.len() as i32).to_le_bytes());
+            c.extend_from_slice(z);
+        }
+        let n = c.len() as i32;
+        c[12..16].copy_from_slice(&n.to_le_bytes());
+        c
+    };
+    let c = chunk(&z);
+    let (r, peak) = peak_during(|| blosc2_decompress_chunk(&c, 32));
+    assert!(peak <= bound(32, &c), "peak {peak} bytes ({r:?})");
+    assert!(r.is_err(), "{r:?}");
+    // The same streams with a window they can use read.
+    z[5] = 0;
+    assert_eq!(
+        blosc2_decompress_chunk(&chunk(&z), 32).unwrap(),
+        vec![7; 32]
+    );
+}
+
+/// xorshift64*: deterministic, so a failure reproduces.
+struct Rng(u64);
+
+impl Rng {
+    fn next(&mut self) -> u64 {
+        let mut x = self.0;
+        x ^= x >> 12;
+        x ^= x << 25;
+        x ^= x >> 27;
+        self.0 = x;
+        x.wrapping_mul(0x2545_F491_4F6C_DD1D)
+    }
+
+    fn below(&mut self, n: usize) -> usize {
+        (self.next() % n.max(1) as u64) as usize
+    }
+
+    /// A size that tends to the edges: small, a power of two, huge.
+    fn size(&mut self) -> i64 {
+        match self.below(6) {
+            0 => self.below(64) as i64,
+            1 => 1 << self.below(31),
+            2 => i32::MAX as i64 - self.below(4096) as i64,
+            3 => (1i64 << self.below(62)) + self.below(8) as i64,
+            4 => MAX_BLOCK - self.below(3) as i64,
+            _ => self.next() as i32 as i64,
+        }
+    }
+}
+
+const MAX_BLOCK: i64 = 536_866_816;
+
+/// One to four edits: bytes, or a size field written little-endian (chunk
+/// headers) or big-endian (frame headers), most often at a header's size
+/// fields.
+fn mutate(rng: &mut Rng, seed: &[u8], data_at: usize) -> Vec<u8> {
+    let mut v = seed.to_vec();
+    for _ in 0..1 + rng.below(4) {
+        let len = v.len();
+        if len < 16 {
+            v.push(rng.next() as u8);
+            continue;
+        }
+        match rng.below(8) {
+            0 => {
+                let i = rng.below(len);
+                v[i] ^= 1 << rng.below(8);
+            }
+            1 => {
+                let i = rng.below(len);
+                v[i] = rng.next() as u8;
+            }
+            2 => {
+                // Frame header: nbytes, cbytes (i64), typesize, chunksize.
+                let x = rng.size();
+                match rng.below(4) {
+                    0 if len >= 38 => v[30..38].copy_from_slice(&x.to_be_bytes()),
+                    1 if len >= 47 => v[39..47].copy_from_slice(&x.to_be_bytes()),
+                    2 if len >= 52 => v[48..52].copy_from_slice(&(x as i32).to_be_bytes()),
+                    _ if len >= 62 => v[58..62].copy_from_slice(&(x as i32).to_be_bytes()),
+                    _ => {}
+                }
+            }
+            3 | 4 => {
+                // A chunk header's nbytes, blocksize or cbytes: in the first
+                // data chunk, or anywhere (the offsets chunk comes last).
+                let at = if rng.below(2) == 0 && data_at + 16 <= len {
+                    data_at + 4 * (1 + rng.below(3))
+                } else {
+                    rng.below(len - 3)
+                };
+                let x = rng.size() as i32;
+                v[at..at + 4].copy_from_slice(&x.to_le_bytes());
+            }
+            5 => v.truncate(rng.below(len)),
+            6 => {
+                let at = rng.below(len);
+                v[at] = [0x10, 0x20, 0x30, 0x40, 0x05, 0x07, 0x02][rng.below(7)];
+            }
+            _ => {
+                let i = rng.below(len - 3);
+                let x = rng.size() as i32;
+                v[i..i + 4].copy_from_slice(&x.to_be_bytes());
+            }
+        }
+    }
+    v
+}
+
+/// Every fixture frame that decodes, with its decoded size.
+fn seeds() -> Vec<(Vec<u8>, usize)> {
+    let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/blosc2");
+    let mut v = Vec::new();
+    for e in std::fs::read_dir(dir).unwrap() {
+        let p = e.unwrap().path();
+        if p.extension().is_some_and(|x| x == "b2f")
+            && let Ok(out) = std::fs::read(p.with_extension("out"))
+        {
+            v.push((std::fs::read(&p).unwrap(), out.len()));
+        }
+    }
+    v.sort();
+    assert!(v.len() >= 20, "fixtures missing");
+    v
+}
+
+fn header_len(frame: &[u8]) -> usize {
+    i32::from_be_bytes(frame[11..15].try_into().unwrap()) as usize
+}
+
+/// Mutated fixture frames, decoded with their HDF5 chunk size as the
+/// limit, and their first chunks on their own: whatever they declare, no
+/// decode holds more than a small multiple of the output and the input.
+#[test]
+fn fuzzed_frames_and_chunks_stay_within_the_allocation_bound() {
+    let _g = lock();
+    let seeds = seeds();
+    let mut rng = Rng(0xb2a1);
+    let mut worst = (0.0f64, String::new());
+    for i in 0..20_000 {
+        let (seed, limit) = &seeds[rng.below(seeds.len())];
+        let f = mutate(&mut rng, seed, header_len(seed));
+        let (r, peak) = peak_during(|| blosc2_decompress(&f, *limit));
+        if let Ok(out) = &r {
+            assert!(out.len() <= *limit, "iteration {i}: output past the limit");
+        }
+        assert!(
+            peak <= bound(*limit, &f),
+            "iteration {i}: peak {peak} bytes for a {limit}-byte chunk from {} bytes ({:?})",
+            f.len(),
+            r.map(|v| v.len())
+        );
+        let ratio = peak as f64 / bound(*limit, &f) as f64;
+        if ratio > worst.0 {
+            worst = (
+                ratio,
+                format!(
+                    "frame iteration {i}: peak {peak}, limit {limit}, input {}",
+                    f.len()
+                ),
+            );
+        }
+    }
+    for i in 0..20_000 {
+        let (seed, _) = &seeds[rng.below(seeds.len())];
+        let at = header_len(seed);
+        let chunk = &seed[at..];
+        let c = mutate(&mut rng, chunk, 0);
+        let limit = 1 << 16;
+        let (r, peak) = peak_during(|| blosc2_decompress_chunk(&c, limit));
+        assert!(
+            peak <= bound(limit, &c),
+            "chunk iteration {i}: peak {peak} bytes from {} bytes ({:?})",
+            c.len(),
+            r.map(|v| v.len())
+        );
+    }
+    eprintln!("worst peak / bound: {:.2} ({})", worst.0, worst.1);
+}
+
+/// Frames built from random header sizes, offsets chunks and B2ND shapes
+/// (chunk and block shapes that pad, special and repeated-value chunks).
+#[test]
+fn random_frames_stay_within_the_allocation_bound() {
+    let _g = lock();
+    let mut rng = Rng(0xb2a2);
+    for i in 0..5_000 {
+        let ts = [1usize, 2, 4, 8][rng.below(4)];
+        let ndim = 1 + rng.below(8);
+        let mut shape = Vec::new();
+        let mut chunks = Vec::new();
+        let mut blocks = Vec::new();
+        for _ in 0..ndim {
+            let s = 1 + rng.below(if ndim > 3 { 4 } else { 40 });
+            let c = if rng.below(8) == 0 {
+                s * (1 + rng.below(4))
+            } else {
+                1 + rng.below(s)
+            };
+            let b = 1 + rng.below(c);
+            shape.push(s as i64);
+            chunks.push(c as i32);
+            blocks.push(b as i32);
+        }
+        let items: usize = shape.iter().product::<i64>() as usize;
+        let limit = items * ts;
+        let meta = nd_meta(&shape, &chunks, &blocks);
+        let ext: usize = ts
+            * chunks
+                .iter()
+                .zip(&blocks)
+                .map(|(&c, &b)| (c as usize).div_ceil(b as usize) * b as usize)
+                .product::<usize>();
+        let nchunks: usize = shape
+            .iter()
+            .zip(&chunks)
+            .map(|(&s, &c)| (s as usize).div_ceil(c as usize))
+            .product();
+        let block_bytes = ts * blocks.iter().product::<i32>() as usize;
+        let chunksize = if rng.below(4) == 0 {
+            rng.size()
+        } else {
+            ext as i64
+        };
+        let nbytes = if rng.below(4) == 0 {
+            rng.size()
+        } else {
+            (nchunks * ext) as i64
+        };
+        let off_n = if rng.below(4) == 0 {
+            rng.size() as i32
+        } else {
+            8 * nchunks as i32
+        };
+        let (data, off) = match rng.below(3) {
+            0 => (Vec::new(), special_offset(1 + rng.below(2) as u8)),
+            _ => {
+                let bs = if rng.below(4) == 0 {
+                    rng.size() as i32
+                } else {
+                    block_bytes as i32
+                };
+                let value: Vec<u8> = (0..ts).map(|_| rng.next() as u8).collect();
+                let n = if rng.below(4) == 0 {
+                    rng.size() as i32
+                } else {
+                    ext as i32
+                };
+                (repeated(&value, n, bs), 0i64.to_le_bytes())
+            }
+        };
+        let offsets = repeated(&off, off_n, off_n.clamp(1, 8));
+        let meta = (rng.below(4) != 0).then_some(meta.as_slice());
+        let f = frame(meta, nbytes, ts as i32, chunksize as i32, &data, &offsets);
+        let (r, peak) = peak_during(|| blosc2_decompress(&f, limit));
+        assert!(
+            peak <= bound(limit, &f),
+            "iteration {i}: peak {peak} bytes for a {limit}-byte chunk ({:?}, shape {shape:?} \
+             chunks {chunks:?} blocks {blocks:?})",
+            r.map(|v| v.len())
+        );
+    }
+}