bench: concurrent-read results on tank, including where we lose

h5py threads stay flat (global lock); clawhdf5 hyperslab reads of deflate
data scale to 1244 MB/s at 16 threads (9.7x h5py threads, 0.89x h5py
processes). Two deficits recorded as open issues: full chunked reads stop
scaling at ~4 threads (chunk cache suspected), and contiguous reads are
4x (full) to 8x (hyperslab) slower than h5py single-threaded.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>

.gitea/workflows/ci.yml

@@ -22,6 +22,9 @@ jobs:
         run: rustup component add rustfmt clippy
       - name: Install thumbv7em-none-eabihf target
         run: rustup target add thumbv7em-none-eabihf
+      - name: Install wasm32-unknown-unknown target
+        # ci-test.sh builds the reader and clawhdf5-wasm for the browser.
+        run: rustup target add wasm32-unknown-unknown
       - name: Install Python interop dependencies
         # The interop suites used to skip silently when python3/h5py were
         # missing, so they never ran in CI. Install them and make a missing
@@ -31,12 +34,18 @@ jobs:
           # cmake builds libz-ng-sys for the opt-in `fast-deflate` (zlib-ng)
           # steps in ci-test.sh; rust:latest does not ship it. The default
           # build (pure-Rust zlib-rs) does not need it.
-          apt-get install -y --no-install-recommends python3 python3-venv cmake
+          # hdf5-tools: h5ls/h5stat/h5dump/h5diff, which the h5rs
+          # (clawhdf5-tools) interop tests compare against.
+          apt-get install -y --no-install-recommends python3 python3-venv cmake hdf5-tools
           python3 -m venv /opt/interop
           /opt/interop/bin/pip install --no-cache-dir h5py numpy netCDF4 xarray hdf5plugin
           echo "/opt/interop/bin" >> "$GITHUB_PATH"
       - name: Show interop library versions
-        run: /opt/interop/bin/python -c "import h5py, netCDF4, hdf5plugin; print('h5py', h5py.__version__, 'HDF5', h5py.version.hdf5_version, 'netCDF4', netCDF4.__version__, 'hdf5plugin', hdf5plugin.version)"
+        # h5dump's version too: the h5rs dump test requires its exact output
+        # (checked against Debian's 1.14.5 in rust:latest and 1.14.6).
+        run: |
+          /opt/interop/bin/python -c "import h5py, netCDF4, hdf5plugin; print('h5py', h5py.__version__, 'HDF5', h5py.version.hdf5_version, 'netCDF4', netCDF4.__version__, 'hdf5plugin', hdf5plugin.version)"
+          h5dump --version
       - name: Run CI script
         env:
           # Name the interpreter outright rather than relying on $GITHUB_PATH

.gitea/workflows/conformance.yml

@@ -0,0 +1,56 @@
+name: Conformance
+# Nightly: read every file of the pinned public HDF5 corpora with clawhdf5 and
+# with h5py/libhdf5 and compare (conformance/run.sh; CONFORMANCE.md explains
+# the method). Fails on any panic, hang, crash or out-of-memory in clawhdf5,
+# and when the ok count drops below conformance/baseline.json or a file the
+# baseline lists as ok stops being ok. The report is printed into the job log;
+# nothing is uploaded (artifact actions are JavaScript, which rust:latest
+# cannot run — see CLAUDE.md).
+on:
+  schedule:
+    - cron: "17 3 * * *"
+  workflow_dispatch:
+jobs:
+  conformance:
+    runs-on: ubuntu-latest
+    container: rust:latest
+    timeout-minutes: 60
+    env:
+      CARGO_NET_RETRY: "10"
+    steps:
+      # Plain git, not actions/checkout (a JavaScript action; see ci.yml).
+      - name: Check out
+        run: |
+          git init -q .
+          git remote add origin "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git"
+          for i in 1 2 3; do git fetch -q --depth 1 origin "${GITHUB_SHA}" && break; sleep 5; done
+          git checkout -q FETCH_HEAD
+      - name: Install h5py, h5dump and the probe's codec libraries
+        # hdf5-tools: h5dump for the CVE-corpus comparison. libaec-dev and
+        # pkg-config: the probe builds clawhdf5-format with `szip` (the core
+        # crates' default build needs neither).
+        run: |
+          apt-get update
+          apt-get install -y --no-install-recommends python3 python3-venv hdf5-tools libaec-dev pkg-config
+          python3 -m venv /opt/conformance
+          /opt/conformance/bin/pip install --no-cache-dir -r conformance/requirements.txt
+          /opt/conformance/bin/python -c "import h5py, hdf5plugin; print('h5py', h5py.__version__, 'HDF5', h5py.version.hdf5_version, 'hdf5plugin', hdf5plugin.version)"
+          h5dump --version
+      - name: Probe unit tests
+        run: cargo test --release --manifest-path conformance/probe/Cargo.toml
+        env:
+          CARGO_TARGET_DIR: conformance/.cache/target
+      - name: Sweep
+        # The corpora come from GitHub (pinned commits, conformance/corpus.txt),
+        # so this job needs a runner that reaches github.com.
+        env:
+          CLAWHDF5_PYTHON: /opt/conformance/bin/python
+        run: bash conformance/run.sh
+      - name: Report
+        if: always()
+        run: |
+          if [ -f CONFORMANCE.md ]; then cat CONFORMANCE.md; else echo "no report was generated"; fi
+          if [ -f conformance/.cache/results/summary.md ]; then
+            echo; echo "---- per-file detail (conformance/.cache/results/summary.md) ----"
+            cat conformance/.cache/results/summary.md
+          fi

BENCHMARKS.md

@@ -482,6 +482,131 @@ The rows and columns of the uncompressed layouts are within 20% (chunked
 column 0.45 -> 0.49 ms, contiguous column 2.55 -> 2.61 ms). This run does not
 explain the slower windows.
 
+## Concurrent reads
+
+### Results (2026-09-26, tank)
+
+Measured on tank (AMD Ryzen 7 7800X3D, 8 cores / 16 threads, 61 GiB, Linux
+7.0) at commit `91644d8`, load average 1.84 when the run started (the
+1-minute figure rose to 3.7 during the runs; that is mostly the benchmark's
+own threads). Warm page cache. clawhdf5 2.7.0 (workspace), h5py 3.16.0 on
+HDF5 2.0.0. Commands exactly as in the **Run** box below; files at their
+defaults (64 datasets of 16384 x 1024 `f32`, 64 MiB each; deflate chunks
+256 x 256, level 4). MB/s is decoded data, the median of the repetitions;
+eff is scaling efficiency against the same tool's 1-thread row.
+
+Each read decoding on its calling thread (`--decode-threads 1`, like h5py):
+
+| layout | mode | threads | clawhdf5 MB/s (eff) | h5py threads MB/s (eff) | h5py processes MB/s (eff) |
+|---|---|---:|---:|---:|---:|
+| deflate | distinct | 1 | 421 (1.00) | 433 (1.00) | 421 (1.00) |
+| deflate | distinct | 4 | 890 (0.53) | 428 (0.25) | 1651 (0.98) |
+| deflate | distinct | 16 | 880 (0.13) | 427 (0.06) | 4424 (0.66) |
+| deflate | same | 1 | 151 (1.00) | 130 (1.00) | 129 (1.00) |
+| deflate | same | 4 | 490 (0.81) | 129 (0.25) | 497 (0.96) |
+| deflate | same | 16 | 1244 (0.52) | 128 (0.06) | 1402 (0.68) |
+| contiguous | distinct | 1 | 2495 (1.00) | 9789 (1.00) | 9169 (1.00) |
+| contiguous | distinct | 16 | 8083 (0.20) | 8096 (0.05) | 12272 (0.08) |
+| contiguous | same | 1 | 624 (1.00) | 5022 (1.00) | 5172 (1.00) |
+| contiguous | same | 16 | 4778 (0.48) | 4411 (0.05) | 37138 (0.45) |
+
+With the default rayon pool decoding inside each read, deflate `distinct`
+is 912 MB/s at 1 thread (2.1x h5py) and 2824 MB/s at 16 (6.6x h5py threads,
+0.64x h5py processes); the other rows are within a few percent of the table
+above. Full tables (2, 4, 8 threads, both decode modes) come from
+`compare_concurrent_read.py` on the JSON files.
+
+What this shows:
+- **h5py threads do not scale** (flat at about 430 MB/s on deflate, every
+  thread count): libhdf5's global lock.
+- **clawhdf5 threads on one `File` do, for hyperslab reads of compressed
+  data:** 1244 MB/s at 16 threads, 9.7x h5py threads and 0.89x h5py
+  processes, without a process pool.
+- **Where clawhdf5 is behind** (open performance bugs, see
+  `docs/known-issues.md`):
+  - *Full reads of chunked datasets stop scaling at about 4 threads*
+    (about 880 MB/s) while h5py processes reach 4424 MB/s. Hyperslab
+    reads, which bypass the `File`'s chunk cache, keep scaling, so the
+    cache (one mutex and one 16 MiB budget per `File`, thrashed by 64 MiB
+    datasets) is the suspect.
+  - *Contiguous reads are slow*: 2.5 GB/s for a single-threaded full read
+    against h5py's 9.8 GB/s (0.25x), and 0.12x for 256 x 256 hyperslabs.
+    Threads close the gap (about 1.0x h5py at 16), but single-thread
+    contiguous I/O is a real deficit.
+
+The question: libhdf5's threadsafe build serialises every API call under one
+global mutex, and h5py holds a global lock around every call too, so threads
+reading through h5py cannot decode in parallel; h5py users scale with
+processes. A clawhdf5 `File` is `Send + Sync`, and nothing on the read paths
+this harness uses (`read_f32`, `read_f32_selection`) takes a library-wide
+lock: the one mutex is the `File`'s chunk cache (keyed per dataset), taken by
+full reads of chunked datasets for each chunk's O(1) lookup and insert, never
+across a decode; hyperslab reads do not use the cache. How does
+decoded throughput scale with threads on one open file, against h5py threads
+and h5py processes on the same files?
+
+Workload (`crates/clawhdf5-bench/src/bin/concurrent_read.rs`; the h5py script
+mirrors it): `<dir>/deflate.h5` and `<dir>/contiguous.h5`, each with 64 `f32`
+datasets of 64 MiB decoded (`[16384, 1024]`; the deflate file chunked
+`256 x 256`, level 4), written by clawhdf5 on first use and reused while
+`manifest.json` matches. The data is a slowly varying ramp plus 8 bits of
+noise per element, every value exact in `f32`, so both harnesses check what
+they read; it deflates about 3.1x (128 MiB -> 40.7 MiB for two 64 MiB
+datasets). For each layout and thread count
+(1, 2, 4, 8, 16; fixed total work per repetition, split among the threads):
+
+- `distinct`: every dataset read in full once, thread `t` taking datasets
+  `t, t + T, ...`;
+- `same`: 1024 random `256 x 256` hyperslabs of `d00` in total, from a seeded
+  splitmix64 stream that both harnesses generate identically.
+
+Reported per row: MB/s of decoded (selected) data from the median of the
+repetitions, and scaling efficiency `MB/s(T) / (T x MB/s(1))`. Each worker
+times itself from a start barrier; a repetition spans the earliest start to
+the latest finish. Page cache: warm by default (each file is read once before
+timing); `--cold` evicts the files with `posix_fadvise(POSIX_FADV_DONTNEED)`
+before every repetition (no root needed; best effort). clawhdf5 opens one
+`File` per repetition, shared by all threads; h5py threads share one
+`h5py.File`; h5py processes (spawned before timing) each open the file inside
+the timed region.
+
+Decode inside a single clawhdf5 read is itself parallel in this binary
+(clawhdf5-format's `parallel` feature, enabled here through clawhdf5-agent;
+it is off in the facade's default features), so a 1-thread clawhdf5 full read
+of the deflate file already uses the whole rayon pool. Run both
+`--decode-threads 1` (each read decodes on its calling thread, like h5py —
+this isolates the API's own scaling) and the default pool.
+
+> **Run** (from the repository root). The default files take about 5.4 GiB
+> of disk (4 GiB contiguous + about 1.3 GiB deflate). Generating them is
+> memory-hungry because `FileBuilder` holds a whole file in memory: peak RSS
+> was 676 MB for `--datasets 2 --mib 64` (2026-09-25, tank,
+> `/usr/bin/time -f %M`), about 5x one file's decoded size, so expect about
+> 21 GB at the defaults (once; later runs reuse the files). Put `--dir` on a
+> real disk, not tmpfs, if `--cold` is to mean anything.
+>
+> ```bash
+> DIR=/path/on/disk/concurrent-read
+> BENCH=crates/clawhdf5-bench/scripts
+> PY=.venv/bin/python   # h5py 3.16 / HDF5 2.0 in this repo
+> cargo build --release -p clawhdf5-bench --bin concurrent_read
+> B=target/release/concurrent_read
+> $B --dir $DIR --json claw-pool.json                    # generates on first run
+> $B --dir $DIR --decode-threads 1 --json claw-1.json
+> $PY $BENCH/concurrent_read_h5py.py --dir $DIR --executor threads   --json h5py-threads.json
+> $PY $BENCH/concurrent_read_h5py.py --dir $DIR --executor processes --json h5py-procs.json
+> $PY $BENCH/compare_concurrent_read.py claw-1.json    h5py-threads.json h5py-procs.json
+> $PY $BENCH/compare_concurrent_read.py claw-pool.json h5py-threads.json h5py-procs.json
+> ```
+>
+> Cold page cache: add `--cold` to every harness command. Smoke test (seconds):
+> `$B --dir /tmp/cr --datasets 4 --mib 1 --threads 1,2,4 --slabs 16 --reps 1`
+> and the same `--threads/--slabs/--reps` to the h5py script.
+
+Other flags (both harnesses): `--threads`, `--reps`, `--slab`, `--slabs`,
+`--seed`, `--modes distinct,same`, `--layouts deflate,contiguous`; sizes
+(`--datasets`, `--mib`) only on the Rust harness, which writes the files.
+
 ## Search harness baseline (v2.3.0)
 
 Produced by `cargo run --release -p clawhdf5-bench --bin search_harness -- --full`

CHANGELOG.md

@@ -2,6 +2,57 @@
 
 ## Unreleased
 
+### Plugin filters (2026-09-26)
+- **LZF, bitshuffle, bzip2 and Blosc read and write, in pure Rust.** Files
+  written by h5py with `compression="lzf"`, or with hdf5plugin's
+  `Bitshuffle`, `BZip2` and `Blosc`, failed with `UnsupportedFilter`. New
+  `clawhdf5-format`/`clawhdf5` features: `lzf` (32000, **on by default**, no
+  dependencies), `bitshuffle` (32008: transpose only, LZ4 and Zstandard
+  modes), `bzip2` (307), `blosc` (32001: Blosc 1 frames with BloscLZ,
+  LZ4/LZ4HC, Snappy, Zlib and Zstandard codecs and byte/bit shuffle;
+  BloscLZ is decoded by a port of c-blosc 1.21's decoder, and cannot be
+  written), and `plugin-filters` for all four. None compiles C: Zstandard is
+  ruzstd, bzip2 is libbz2-rs-sys. Write with `DatasetBuilder::with_lzf()`,
+  `with_bitshuffle(..)`, `with_bzip2(..)`, `with_blosc(..)` or
+  `with_plugin_filter(PluginFilter::..)`; `ChunkOptions` gains a `plugin`
+  field (**breaking** for code that builds `ChunkOptions` with a struct
+  literal and no `..Default::default()`). Tested both ways against h5py 3.16
+  + hdf5plugin 7.1 over 1-3-D shapes with partial edge chunks, 1-8-byte
+  types in both byte orders and incompressible data
+  (`crates/clawhdf5/tests/plugin_filters_interop.rs`). Conformance: 573 of
+  697 files ok (was 569) — h5ex_d_lzf/bshuf/bzip2/blosc.
+- **Filter registry.** Filters are looked up by ID in
+  `clawhdf5_format::filter_registry` instead of a `match`: the built-in
+  table (per build), then codecs registered at run time with
+  `register_filter(id, codec)` — a decoding closure or a `FilterCodec` that
+  can also encode. Built-in IDs cannot be overridden; a registered decoder's
+  output is held to the chunk-size bound. Unknown IDs still fail with
+  `UnsupportedFilter(id)`, whose message now names known filters and the
+  missing feature ("unsupported filter: 32026 (Blosc2, not implemented by
+  clawhdf5)").
+- **Not implemented:** Blosc2 (32026) and ZFP (32013) remain a clear error.
+- **Wrong data: a chunk that decodes short read as zeros** (pre-existing, every
+  filter). HDF5 stores every chunk at the full chunk size, so a filter
+  pipeline that decodes to fewer bytes means a corrupt chunk; every chunk
+  reader (full, cached, selection, parallel, partial) padded it with zeros.
+  It is now an error naming the chunk ("chunk at [16] decoded to 16 bytes,
+  expected 32"), via the new `filters::decompress_chunk_exact`. libhdf5
+  returns the rest of such a chunk uninitialised, or fails when the filter
+  checks. A Blosc frame declaring no data for a non-empty chunk is an error
+  too. Legitimate edge chunks are unaffected (they are stored full-size,
+  filtered or not); conformance is unchanged at 573 of 697, with no file
+  changing class.
+- **Crash: a hostile Blosc chunk panicked** in builds with overflow checks
+  (debug builds, `cargo test`, `maturin develop`): a frame size below the
+  16-byte header underflowed. It is now an error. Every new decoder (LZF,
+  bitshuffle, bzip2, Blosc/BloscLZ) is fuzzed with random and mutated frames
+  in the unit tests.
+- **`register_filter(32023, ..)` works with the `pcodec` feature.** 32023 is
+  Granular BitRound's ID; the built-in entry there only reads clawhdf5
+  <= 2.7.0's pcodec chunks (filter name `"pcodec"`), so a registered codec now
+  handles every other chunk with that ID, and writes. It was refused as
+  "built in".
+
 ### Upgrade Notes
 - **HDF5 correctness audit (2026-09-25).** A sweep of 686 public files (the
   libhdf5 test files, the HDF Group's CVE reproducers, pyfive, netcdf-c,
@@ -112,6 +163,65 @@
   `quantized_index = false`, or pass `create --f32-index` to the CLI, to opt
   out. The CLI's `--quantized-index` is still accepted but is now a no-op.
 
+### Tools
+- New crate **`clawhdf5-tools`** with the binary **`h5rs`**: HDF5
+  command-line tools without libhdf5, built only on the `clawhdf5` facade
+  and `clawhdf5-format` (no C, so it also builds as a static musl binary).
+  - `h5rs ls [-r] [-v] FILE[/path]` lists objects like h5ls (its first two
+    columns are h5ls's text on the test files) plus the datatype; `-v` adds
+    address, link count, layout and chunk index, chunk size, storage,
+    filters, datatype and attributes.
+  - `h5rs dump [--json] [-A] [-p] [-d PATH] FILE` prints DDL text that is
+    byte-identical to h5dump 1.14.6's (and to Debian's 1.14.5, which CI
+    uses) on the test files (all layouts and
+    chunk indexes, v1/v2 groups, compound, enum, strings, links, named
+    types, attributes; null-padded strings show their NULs at any depth),
+    or JSON in the HDF Group's hdf5-json layout (schema in the crate
+    README). Nested compounds print inline and `long double` values as
+    errors (exit 1); both are listed in the README.
+  - `h5rs stat FILE` reports h5stat's object, link, rank, layout, filter,
+    attribute, raw-data and file-size figures (equal to h5stat's on the test
+    files); metadata space is one figure, not broken down.
+  - `h5rs diff [-r] [-q] [-n N] [-d D] [-p R] [--follow-symlinks] A B [OBJ1
+    [OBJ2]]` (option names as h5diff's: `-c` is `--compare`, the count is
+    `-n`/`--count=N`) compares objects, kinds, datatypes, shapes, attributes, values and link
+    targets; exit status 0/1/2 as h5diff's. Soft links are compared by
+    target path, as h5diff's default, or with `--follow-symlinks` by the
+    objects they lead to (external links are never followed). Every path is
+    compared, including every name of a hard-linked object and the members
+    of a hard-linked group; with a `-d`/`-p` tolerance, integers are
+    compared exactly in integer arithmetic (no loss above 2^53), and a `-p`
+    below the f64 epsilon compares exactly, as h5diff's. Objects that cannot
+    be compared count as a difference (h5diff exits 0 for them), and NaN
+    equals NaN.
+  - `h5rs check [--data] FILE` is a structural validator: it walks every
+    object, parses every header message, verifies the checksums of every
+    version 2+ structure it meets (superblock, object headers and
+    continuation chunks, v2 B-tree nodes, fractal heap headers and — which
+    the library's reads do not — every direct and indirect heap block, and
+    extensible/fixed array chunk indexes), checks each chunk index against
+    its dataset (aligned, in-extent, unique, plausibly sized chunks), and
+    that raw data lies inside the file without overlaps. Every problem is
+    printed with its address; exit 1 when there are any. libhdf5's h5check
+    reads only the 1.8 format. On the conformance corpus it passes all 418
+    files that both clawhdf5 and h5py read in full, and `check --data` flags
+    134 of the 150 CVE and fuzzer files of the `cve_hdf5` corpus (tank,
+    2026-09-26). `--data` also follows variable-length data into its global
+    heap collections and reports a damaged one at its address. It inherits
+    the library's tolerance, though: 9 of the 16 it passes are files h5dump
+    1.14.6 rejects (see `docs/known-issues.md`, header checks).
+  - Values over `--max-bytes` (default 1 GiB) are reported instead of read;
+    a panic is caught and reported as an internal error (exit 3).
+    `scripts/h5rs-fuzz.sh` runs every subcommand over a corpus (default the
+    CVE reproducers, optionally with byte-flipped copies) with overflow
+    checks, a timeout and a memory limit, and fails on any panic, crash or
+    hang; `scripts/h5rs-check-ok-files.sh` runs `check --data` over the
+    fully-read conformance files.
+  - Because the library does not verify fractal heap block checksums when
+    it reads a dense group's links or dense attributes, `h5rs` verifies a
+    heap's blocks before reading from it and refuses a damaged one, as
+    libhdf5 does, instead of printing what the damaged block holds.
+
 ### Signing
 - `clawhdf5-agent`: **Ed25519-signed checkpoints** — the README's
   "cryptographically verifiable memory", now true. With
@@ -195,8 +305,39 @@
   README claimed but nothing measured.
 - `footprint_bench` reports whether it built `float16` or `f32` stores and
   takes `--f32`; it had kept printing "f32" after the default changed.
+- New `concurrent_read` harness, with an h5py counterpart
+  (`crates/clawhdf5-bench/scripts/concurrent_read_h5py.py`, threads or
+  processes) and `compare_concurrent_read.py`: decoded read throughput and
+  scaling efficiency at 1-16 threads on one open file, full reads of distinct
+  datasets and random hyperslabs of one dataset, deflate and contiguous, warm
+  or `--cold` page cache, JSON output. Not yet measured — `BENCHMARKS.md`
+  ("Concurrent reads") has the commands and no numbers.
 
 ### Interop
+- **h5py could not open chunked datasets we wrote with a chunk dimension
+  from 65 536 to 16 777 215.** A version-4 layout must store its chunk
+  dimensions in the fewest bytes that hold the largest (3 for 70 000);
+  the writer rounded 3 up to 4, and HDF5 2.0.0 (h5py 3.16) refuses that
+  ("stored chunk dimension encoding length does not match value calculated
+  from chunk dimensions"). Newer libhdf5 and clawhdf5 read those files; new
+  files use the exact width. Test: `we_write_chunk_dimensions_in_the_fewest_bytes`.
+- **Conformance sweep in the repo** (`conformance/`, report in
+  `CONFORMANCE.md`). `conformance/run.sh` fetches eight public HDF5 corpora
+  pinned by commit (libhdf5's test files, the HDF Group's CVE reproducers,
+  pyfive, netcdf-c, netcdf4-python, h5wasm, h5py, xarray-data) into a
+  gitignored cache, reads every file with clawhdf5 and with h5py/libhdf5 (and
+  the CVE files with h5dump) under a timeout and memory limit, compares them
+  object by object and regenerates the report — about 30 s once the corpus is
+  cached. A nightly Gitea job (`.gitea/workflows/conformance.yml`) runs it and
+  fails on any panic, hang, crash or out-of-memory, or when a file in
+  `conformance/baseline.json` stops reading identically. First report, on
+  42b81d9: 467 of 697 files identical to h5py, 123 our-error, 15 mismatch
+  (2 of them an h5py bug), 92 that libhdf5 cannot read, no panics, hangs or
+  crashes. Compared with the ad-hoc audit sweep, the probe now compares
+  N-Bit floats (and integers with a bit offset) as the values libhdf5
+  converts them to rather than raw file bytes — 8 files that were reported as
+  mismatches read identically — and the reference side no longer flips
+  between runs when libhdf5 aborts while freeing h5py objects.
 - `clawhdf5-format`: **every `f32` dataset was unreadable by h5py and
   libhdf5.** The float datatype encoder hard-coded the sign bit's position to
   63, correct only for `f64`; libhdf5 validates it and refused the dataset. It
@@ -236,6 +377,12 @@
   - A pipeline with Fletcher32 ahead of the compressor (h5py
     `set_fletcher32()` then `set_deflate()`) no longer fails with "deflate:
     output exceeds size limit".
+- `clawhdf5-format`: **HDF5 1.4/1.6-era files are readable.** Data Layout
+  message versions 1 and 2 (compact, contiguous, and chunked through the
+  version-1 B-tree) failed with `InvalidLayoutVersion` — 84 of the 686 files in
+  the 2026-09-25 audit sweep, 205 datasets. They now read as libhdf5 does;
+  checked byte for byte against h5py on HDF5's own test files
+  (`tests/legacy_format_interop.rs`).
 
 ### Storage
 - `clawhdf5-format`: **half-precision datasets.**
@@ -257,6 +404,32 @@
   infinity; batches are all or nothing. CLI: `create --float16`. See
   `BENCHMARKS.md`, "float16 embedding storage".
 
+### Browser (WebAssembly)
+- **New crate `clawhdf5-wasm`:** the reader compiled to
+  `wasm32-unknown-unknown` with a wasm-bindgen JavaScript API —
+  `open(bytes)`, `list`, `info`, `attrs`, `read`, `readHyperslab` — returning
+  typed arrays of the stored width (`BigInt64Array` for 64-bit integers),
+  string arrays for strings and enums, and a thrown `Error` for types with no
+  typed-array form (compound, reference, opaque, VL sequences) or filters the
+  build lacks (Zstd, SZIP). Read-only; the file is held in memory.
+- **`examples/wasm-viewer/`:** a drop-a-file HDF5/NetCDF-4 viewer page (tree,
+  type/shape/attributes, values paged as hyperslabs; `?file=&path=` opens a
+  URL). `build.sh` produces the package; `test/run.sh` checks it under Node
+  (251 checks against values h5py/libhdf5 read back from an h5py- and a
+  netCDF4-written file) and renders the page in headless Chromium. Size,
+  measured 2026-09-26 on tank (`gzip -9 -n`): 627,501 B of wasm, 191,639 B
+  gzipped, plus 21,826 B (4,487 B) of JS glue; h5wasm 0.10.3's embedded wasm
+  is 3,544,184 B (907,096 B) — full libhdf5, so not equal functionality. See
+  `examples/wasm-viewer/README.md`.
+- The facade's read path already built for `wasm32-unknown-unknown` (nothing
+  needed gating); `ci-test.sh` now builds it (`--no-default-features`) and
+  lints `clawhdf5-wasm` for that target, and CI installs the target. The Node
+  and browser tests run in `ci-test.sh` only where `node` and `wasm-bindgen`
+  exist (not the CI container); CI checks the same expectations natively
+  (`clawhdf5-wasm`'s `h5py_interop` test).
+- `Dataset::raw_datatype()` (facade) returns the full stored datatype, for
+  decoding `read_selection` bytes with `clawhdf5_format::data_read`.
+
 ### Build
 - **Pure-Rust default.** `clawhdf5-format`, `clawhdf5-filters` and the
   `clawhdf5` facade default to the `zlib-rs` deflate backend; `fast-deflate`
@@ -273,6 +446,168 @@
 - CI keeps zlib-ng building and tested; the arm64 job no longer needs cmake.
 
 ### Correctness
+- **Corrupt files libhdf5 refuses are now refused instead of read.** On the
+  HDF Group's CVE reproducers, 18 objects that libhdf5 (HDF5 2.0, through
+  h5py) refuses to open were read by clawhdf5, some as wrong data (a chunk
+  dimension of 0 read as all fill values; chunks read at offsets off the
+  chunk grid). The
+  parser now makes libhdf5's checks, with libhdf5's error text:
+  - object headers (`FormatError::InvalidObjectHeader`): every message of a
+    v1 chunk is read and more than the prefix's count is refused (the rest
+    used to be dropped); v1 message sizes must be multiples of 8 and a v1
+    chunk cannot end in a gap; a message running past its chunk is an error
+    (it used to end the chunk quietly); contradictory message flags; a
+    message of a class that cannot be shared flagged shareable; a
+    reference-count message in a v1 header; malformed continuation,
+    reference-count and modification-time messages; unknown v2 header
+    flags.
+  - datatypes (`FormatError::InvalidDatatype`): size 0; integer bits outside
+    the type; float exponent/mantissa outside the type, empty or
+    overlapping; a compound with no members, a member outside the compound,
+    a duplicate name or overlapping members; an enum whose size differs from
+    its base type's or with an empty name; array rank over 32 or a zero
+    dimension; an opaque tag length that is not a multiple of 8; in a
+    version-1 (unchecksummed) header, a numeric type that leaves more than
+    half its bits unused (`Datatype::parse_in_header`,
+    `Datatype::check_unused_bits`). A v1/v2 float's class bit 6 was read as
+    VAX byte order; libhdf5 ignores it before version 3, and so does this.
+    The overlap check measures each earlier member by its stored size, as
+    libhdf5 does, so a variable-length member (4 + offset size + 4 bytes)
+    in a file with 4-byte offsets does not overlap the member after it.
+  - chunked layouts (`FormatError::InvalidChunkDimensions`): a zero chunk
+    dimension, a chunk rank that does not match the dataspace, a chunk of
+    4 GiB or more indexed by a v1 B-tree (layout version 3 or earlier;
+    0x80000000-sized chunks hung the reader — layout versions 4 and 5 allow
+    larger chunks, and HDF5 2.0 writes them), an element size in the
+    layout that differs from the datatype's stored size (the chunks were
+    laid out with the wrong element size), and v1 B-tree
+    chunk keys whose offsets are not multiples of the chunk dimensions,
+    including the keys that only bound a node
+    (`chunked_read::collect_chunk_info_checked`).
+  - truncated files (`FormatError::TruncatedFile`, `Superblock::data_end`):
+    a file shorter than the end of file its superblock records is refused
+    ("truncated file"), and nothing past that end is read. Every reader
+    does this: `File`, `LazyFile` and `MmapFile`, and in `clawhdf5-io`
+    `NativeVol` (at `open`, and on read for `from_bytes`),
+    `AsyncHDF5File` and `MpiVol` (the MPI path is not built in CI: it
+    needs an MPI installation).
+  - the writer: `FileWriter::finish()` / `FileBuilder::finish()` refuse a
+    datatype the reader would refuse (`FormatError::SerializationError`,
+    "datatype cannot be written: ..."), such as a compound with a repeated
+    field name or no fields, or an enum member with an empty name
+    (`CompoundTypeBuilder` and `EnumTypeBuilder` build them without
+    complaint). These were never valid HDF5 — h5py refuses them — and
+    clawhdf5 wrote them, which made files it could not read back.
+
+  Checks newer libhdf5 releases make but HDF5 2.0 does not (bit-field
+  offsets, the variable-length kind, array sizes) are left out, so files
+  h5py opens still open. Two libhdf5 checks are skipped on purpose because
+  clawhdf5 up to v2.7.0 wrote files that fail them without being wrong:
+  the sign bit of every float at position 63, and a size-0 string type for
+  an empty-string attribute (new fixtures written by v2.7.0 guard this).
+  Conformance: 569 -> 571 ok (h5stat_err_refcount.h5,
+  h5clear_fsm_persist_less.h5), and 17 of the 18 CVE objects now fail as in
+  libhdf5 (see `docs/known-issues.md` for the one left), as do 10 files
+  h5py refuses as truncated. Tests:
+  `header_validation_interop.rs` (h5py writes, the test damages a copy, both
+  libraries must refuse it), `legacy_writer_files.rs`, and unit tests next
+  to each check. **Breaking (format crate):** `FormatError` gained
+  `InvalidObjectHeader`, `InvalidDatatype`, `InvalidChunkDimensions` and
+  `TruncatedFile`; an exhaustive `match` on it needs the new arms.
+- **Chunked datasets whose chunk dimensions take 3, 5, 6 or 7 bytes did not
+  open.** A version-4 layout (`libver="latest"`) stores each chunk dimension
+  in the fewest bytes that hold the largest one, so a chunk dimension from
+  65 536 to 16 777 215 (e.g. h5py `chunks=(70000,)`) takes 3 bytes; only 1, 2,
+  4 and 8 were read, and the rest failed with `UnexpectedEof`. Widths 1-8 are
+  read now, and 0 or more than 8 is refused as libhdf5 refuses it. A width
+  larger than needed is accepted: HDF5 2.0.0 refuses one ("stored chunk
+  dimension encoding length does not match"), but libhdf5 since
+  HDFGroup/hdf5@e124c36 (2026-06-05) reads it, and clawhdf5 itself wrote such
+  layouts.
+- `clawhdf5-format` VDS: variable-length and reference data from a source in
+  another file is refused. Those elements are global-heap IDs and object
+  addresses in the source file; copied into the virtual dataset they would
+  be decoded against the wrong file and name another object.
+- `clawhdf5-agent`: a store whose `/meta` has an attribute that cannot be
+  decoded fails to open (`MemoryError::Schema`). With `attrs()` now leaving
+  unreadable attributes out, it would otherwise have opened with defaults in
+  place of its settings (`float16`, `compression`, the WAL mark, ...).
+- `clawhdf5-format` reader: an old-style group whose local heap has a free
+  list pointing outside the heap was listed with names read from the broken
+  heap (garbage names on `cve-2021-36977.h5` once its user block was
+  applied). libhdf5 refuses such a heap ("bad heap free list"); so do we now,
+  with `FormatError::InvalidLocalHeapFreeList`. As in libhdf5 the free list
+  is checked when the first name is read (`LocalHeap::validate_free_list`,
+  new), so an empty group with a damaged heap still lists as empty.
+- **Files with a user block** (`h5py.File(..., userblock_size=N)`, `h5jam`;
+  the superblock at 512, 1024, …) could not be read: every address in the
+  file is relative to the superblock, but it was applied from byte 0
+  (`InvalidObjectHeaderVersion` on the root group). `File` (mmap, buffered,
+  `from_bytes`), `MmapFile`, `LazyFile`, `AsyncHDF5File`, the VOL readers,
+  the HNSW loader and external VDS sources now view the file from the
+  superblock on, using the signature's position as the base address as
+  libhdf5 does; `user_block_size()` reports the user block (h5py's
+  `userblock_size`), and `as_bytes()` returns the bytes from the superblock
+  on. **Breaking (format crate):** `Superblock::parse` refuses a non-zero
+  signature offset with `FormatError::UserBlockNotStripped`, since the
+  addresses it returns would be applied to the wrong bytes; pass the slice
+  from `signature::split_user_block` (new) and parse at offset 0.
+- `clawhdf5-format` reader: version-1 shared messages (HDF5 1.6-era files,
+  e.g. a dataset using a committed datatype in libhdf5's `tcompound.h5`)
+  read the heap-offset field of the embedded symbol-table entry as the
+  target address and failed with `InvalidObjectHeaderVersion`. The address
+  is now read after it, as libhdf5 does. **Breaking (format crate):**
+  `shared_message::parse_shared_ref` takes `length_size`. A reference whose
+  target header has no message of the referenced type is now
+  `FormatError::SharedMessageTargetMissing` instead of returning the first
+  other message found there (which decoded as garbage).
+- `clawhdf5-format` reader: array members of version-1 compound datatypes
+  (HDF5 1.6-era files, e.g. libhdf5's `tcompound.h5`) were read as a single
+  element: a `[4] i32` member came back as one `i32`, with the wrong size.
+  The legacy per-member dimension fields are now decoded into an array type,
+  as libhdf5 does; more than four dimensions, or a zero-sized one, is an
+  error.
+- `clawhdf5-format` virtual datasets (VDS), checked against HDF5 2.0 through
+  h5py (`crates/clawhdf5/tests/vds_interop.rs`):
+  - **Wrong data:** elements no mapping supplies — unmapped regions, and
+    mappings whose source file or dataset is missing — read as 0 instead of
+    the virtual dataset's fill value (e.g. h5py `fillvalue=-1`). Assembly moved
+    to the new `vds` module: `vds::read_virtual_dataset` takes the fill value
+    and a resolver that can refuse a name (`VdsFileResolver`), and `File`
+    passes the dataset's fill value. A missing source *dataset* read as an
+    error; it is fill now, as in libhdf5. Source datasets are read with their
+    own fill value for unallocated chunks, and a source whose datatype differs
+    from the virtual dataset's is an error (libhdf5 converts; we do not).
+    `File` now refuses a source name that leaves the virtual file's directory
+    (`../x.h5`, absolute paths), or any external source of a `File::from_bytes`
+    file, with an error — these used to read as fill.
+    **Behaviour change:** the raw-read API (`read_raw_data_full*`), which has
+    no fill value, now returns an error for a virtual dataset with unmapped
+    elements instead of zeros.
+  - Unlimited and printf-style mappings are supported (all 7 VDS files in the
+    libhdf5 test set are such mappings, e.g. Eiger/Percival detector layouts).
+    `%b` in a source file or dataset name is the block number and `%%` a
+    literal `%` (other `%` sequences are an error, as in libhdf5); block `j`
+    is read from the source named with `j`, probing from 0 up to the first
+    missing source. Unlimited source/virtual selections cover as much as the
+    source's current extent fills, including a partial last block. As
+    libhdf5 does on `H5Dget_space`, the extent is recomputed from the sources
+    present (default "last available" view, printf gap 0) —
+    `vds::virtual_dataset_extent`, used by `Dataset::shape()` — so e.g.
+    `vds-eiger.h5` is `[5, 10, 10]`, not its stored `[20, 10, 10]`. A source
+    stored in the other byte order is byte-swapped (libhdf5 converts);
+    other type conversions remain an error.
+  - Hyperslab selection versions 1 and 2 were refused ("only version-3
+    hyperslab selections are supported"). Version 1 is what libhdf5 writes for
+    every VDS created with the default format bounds (h5py's default), so
+    those could not be read at all; version 2 is its encoding of an unlimited
+    selection. Both are decoded now, as are irregular hyperslabs (a union of
+    blocks, read in row-major order as libhdf5 iterates them).
+    `SerializedSelection` exposes the raw form, including unlimited counts.
+  - The version-1 mapping list HDF5 2.0 writes (low version bound 2.0) was
+    misparsed: each entry's flags byte was read as the start of the source
+    file name, and names shared with an earlier entry (stored as that entry's
+    index) were not followed. Now decoded as `H5D__virtual_load_layout` does.
 - `clawhdf5-format` reader — **values returned wrong with no error:**
   - Fixed Array and Extensible Array chunk indexes were laid out by the
     dataset's current shape instead of its max shape (23 libhdf5 test files,
@@ -294,11 +629,77 @@
   - Two threads reading two chunked datasets through one `File` could get each
     other's chunks (the shared chunk cache was switched between datasets
     across separate lock acquisitions). The cache is now keyed by dataset.
+  - Compound datatype version 1 members with legacy array dimensions (HDF5
+    before 1.4, which had no array class) were read as a single scalar at
+    the member's offset; they are now array members, as in libhdf5
+    (`tarrold.h5`, `tcompound.h5`). Only reachable once layout versions 1/2
+    were readable, since the files that use it are that old.
+- `clawhdf5-format` reader — errors on valid files: a version-1 shared
+  message (a committed datatype in HDF5 1.4/1.6-era files) was read as if the
+  object header address followed the reserved bytes; it follows a link-name
+  offset (the reference is an old-style symbol table entry), so the reader
+  followed the name offset and failed with `InvalidObjectHeaderVersion`
+  (`tcompound.h5`). New `shared_message::parse_shared_ref_sized` takes the
+  superblock's length size; `parse_shared_ref` assumes it equals the offset
+  size.
 - `clawhdf5-format` reader — errors on valid files: enum and bool datasets
   through the numeric readers; the "don't filter partial edge chunks" layout
   flag; Fletcher32 ahead of deflate (NetCDF-4's order). Unknown-message flags
   follow libhdf5 (`tbogus.h5`): "fail if unknown" is refused, "fail if unknown
   and writing" is ignored by a reader.
+- `clawhdf5-format` reader — dense groups and attributes (links or
+  attributes kept in a fractal heap indexed by a v2 B-tree):
+  - A link heap larger than the root indirect block's direct rows (512 KiB
+    with libhdf5's defaults: a few thousand long link names, or ~20 000 short
+    ones) could not be listed: child indirect blocks were given the wrong
+    number of rows, so every link stored in one was unreachable.
+  - v2 B-trees of depth 3 or more (a dense group of ~22 000+ links) were
+    misparsed: internal-node child pointers were read with widths from an
+    estimate instead of libhdf5's per-depth record capacities, and the
+    listing failed. The same B-tree code indexes dense attributes, shared
+    messages and chunks.
+  - Fractal-heap "huge" objects (larger than the heap's managed-object
+    limit, 4 KiB by default — e.g. an 8 KiB dense attribute or a link with a
+    very long name) and "tiny" objects are now read; the ID type was taken
+    from the wrong bits (6-7, the version, instead of 4-5), so a huge object
+    failed and took every attribute on its object down with it (NetCDF-4
+    files such as netcdf4-python's `issue671.nc`). Huge objects are found
+    directly from the ID or through the huge-object v2 B-tree, filtered or
+    not.
+  - Heaps with an I/O filter pipeline (a group created with a filter on its
+    creation property list compresses its link heap) are now read: the
+    header's pipeline was skipped with the wrong size, so its checksum was
+    looked for in the wrong place, and filtered direct blocks were read raw.
+  - A user-defined link (link class 65-255, e.g. 187 in libhdf5's
+    `tall.h5`/`tudlink.h5`) made its whole group unlistable. Such links
+    cannot be followed without the application that registered the class, so
+    they are now left out of `datasets()`/`groups()` and path lookup, as h5py
+    leaves out links it cannot open; reserved link types are still an error.
+- `clawhdf5` — soft links are listed, as h5py lists them: `datasets()` and
+  `groups()` on `Group`/`MmapGroup`/`LazyGroup` include each soft link under
+  its own name as the kind of object it resolves to, and `dataset(name)` /
+  `group(name)` open through it. Relative targets resolve from the group
+  holding the link. Dangling or cyclic soft links, external links and
+  user-defined links are left out (h5py lists their names but cannot open
+  them). Previously soft links were missing from the listings, and in
+  old-style (symbol table) groups a soft link made the listing fail. New
+  `group_v2::resolve_group_children` / `resolve_path_from` and
+  `group_v1::v1_soft_links` in `clawhdf5-format`.
+- `clawhdf5` — one unreadable attribute no longer fails `attrs()` for every
+  attribute on its object: it is left out of the map, and the new
+  `attrs_with_errors()` (on every group and dataset handle) returns the map
+  plus one error per attribute left out. Returned values are always complete.
+  An error in the attribute index itself (attribute info message, dense heap
+  header or B-tree) still fails the call. `clawhdf5-format` gains
+  `attribute::extract_attributes_tolerant`; `extract_attributes_full` stays
+  strict.
+- `clawhdf5-format` reader — files with shared object header messages
+  (SOHM, `H5Pset_shared_mesg_index`): a datatype, dataspace, filter pipeline
+  or attribute stored in the file's SOHM heap failed with "invalid shared
+  message version: 2" — only shared fill values loaded the SOHM table — so
+  such files' datasets and attributes could not be read.
+  `shared_message::resolve_shared_message` now loads the table when a
+  reference needs it (36 cases of the audit's read matrix).
 - `clawhdf5-format` writer — **files libhdf5 rejects or reads wrong:**
   - Extensible Array (one unlimited dimension): chunks from index 244 on were
     written but never indexed and read as 0, by libhdf5 and by us.

CLAUDE.md

@@ -5,13 +5,13 @@ Pure-Rust HDF5 format implementation with HNSW vector search, WAL-backed persist
 
 ## Architecture
 
-Cargo workspace with 16 crates under `crates/` (plus `libaec-sys`, an internal FFI bindings crate for the optional `szip` feature):
+Cargo workspace with 18 crates under `crates/` (plus `libaec-sys`, an internal FFI bindings crate for the optional `szip` feature):
 
 | Crate | Role |
 |-------|------|
 | `clawhdf5-format` | HDF5 binary spec parser (superblock, B-tree, heap) — also holds shared type definitions and physical constants |
 | `clawhdf5-io` | Read/write implementation |
-| `clawhdf5-filters` | Deflate backends (zlib-rs, zlib-ng, Apple Compression); the HDF5 filter pipeline and the other codecs (LZ4, Zstd, SZIP, N-Bit, scale-offset, pcodec) live in `clawhdf5-format`. No Blosc. |
+| `clawhdf5-filters` | Deflate backends (zlib-rs, zlib-ng, Apple Compression); the HDF5 filter pipeline, the filter registry (`clawhdf5_format::filter_registry`) and the other codecs (LZ4, Zstd, SZIP, N-Bit, scale-offset, pcodec, and the pure-Rust plugin filters LZF, bitshuffle, bzip2, Blosc 1) live in `clawhdf5-format`. No Blosc2 or ZFP. |
 | `clawhdf5-derive` | Proc-macro derive for HDF5-serializable structs |
 | `clawhdf5` | Main facade crate |
 | `clawhdf5-netcdf4` | NetCDF-4 compatibility layer |
@@ -21,9 +21,11 @@ Cargo workspace with 16 crates under `crates/` (plus `libaec-sys`, an internal F
 | `clawhdf5-accel` | CPU SIMD acceleration path |
 | `clawhdf5-migrate` | SQLite → HDF5 agent-memory migration |
 | `clawhdf5-android` | Android JNI bindings |
-| `clawhdf5-cli` | Command-line interface |
+| `clawhdf5-cli` | Command-line interface (agent memory) |
+| `clawhdf5-tools` | `h5rs`: pure-Rust HDF5 tools — `ls`, `dump` (DDL / hdf5-json), `stat`, `diff`, `check` (structural + checksum validator) |
 | `clawhdf5-napi` | Node.js native addon bindings |
 | `clawhdf5-py` | PyO3 Python bindings |
+| `clawhdf5-wasm` | WebAssembly (wasm-bindgen) reader for the browser; demo in `examples/wasm-viewer/` |
 | `clawhdf5-bench` | Benchmark suite |
 
 ## Key Features
@@ -149,6 +151,14 @@ Cargo workspace with 16 crates under `crates/` (plus `libaec-sys`, an internal F
   `MemorySource` for this bookkeeping is inferred from the caller-supplied
   `source_channel` string (a heuristic, not an authenticated trust boundary).
 - GPU-accelerated vector distance computation (`clawhdf5-gpu`, wgpu); HDF5 I/O itself is CPU-only
+- Browser: `clawhdf5-wasm` (wasm-bindgen, read-only, file held in memory;
+  no Zstd/SZIP since they link C) and the `examples/wasm-viewer/` page.
+  `examples/wasm-viewer/test/run.sh` builds the package (needs the
+  `wasm-bindgen` CLI at the crate's exact version) and tests it under Node
+  and headless Chromium (a Playwright download in `~/.cache/ms-playwright`
+  on tank); the CI container has neither, so CI runs the native
+  `clawhdf5-wasm` `h5py_interop` test on the same fixture. Size numbers are
+  in the example's README.
 - Python and Node.js bindings for cross-language use
 - NetCDF-4 compatibility for scientific data interop
 
@@ -188,6 +198,16 @@ cargo run -p clawhdf5-cli -- --help
 # create, save, search, recall, stats, flush-wal, agents-md, export, snapshot subcommands
 ```
 
+### HDF5 tools (`h5rs`, crate `clawhdf5-tools`)
+```bash
+cargo run -p clawhdf5-tools -- ls -r file.h5    # also dump [--json], stat, diff, check
+bash scripts/h5rs-fuzz.sh                       # every subcommand over the CVE corpus: no panic/crash/hang
+bash scripts/h5rs-check-ok-files.sh --data      # check passes every fully-read conformance file
+```
+Its interop tests compare against h5ls/h5stat/h5dump/h5diff (Debian
+`hdf5-tools`, installed in CI); `dump` must stay byte-identical to h5dump on
+the test files.
+
 ### Python bindings
 ```bash
 cd crates/clawhdf5-py

CONFORMANCE.md

@@ -0,0 +1,298 @@
+# clawhdf5 conformance report
+
+Every HDF5 file of eight public corpora (pinned by commit) is read twice — by
+clawhdf5 (`conformance/probe`, the same `clawhdf5-format` calls the facade
+makes) and by h5py/libhdf5 (`conformance/ref.py`) — and the two readings are
+compared object by object: the set of hard-linked objects, each dataset's and
+attribute's shape, and a SHA-256 of its values in a canonical encoding. The
+CVE corpus is also run through `h5dump`. Each side runs under a timeout and an
+address-space limit, so a hang, crash or runaway allocation is recorded, not
+fatal. This file is generated by `conformance/run.sh`; do not edit it by hand.
+
+## Run
+
+| | |
+|---|---|
+| date | 2026-09-26 06:50 UTC |
+| clawhdf5 commit | `72306c601399748616bc9d061be2ebc4c1bea9e0` |
+| machine | `tank`: AMD Ryzen 7 7800X3D 8-Core Processor, 16 CPUs, 61 GiB, Linux 7.0.0-34-generic x86_64 |
+| command | `conformance/run.sh --no-fetch --update-baseline` |
+| rustc | rustc 1.98.1 (48a229cea 2026-09-01) |
+| reference | h5py 3.16.0, HDF5 2.0.0, numpy 2.5.3, hdf5plugin 7.1.0, Python 3.14.4 |
+| h5dump | Version 1.14.6 (CVE corpus only) |
+| limits | 20 s timeout (SIGKILL), 4096 MiB address space, per process; 16 files in parallel |
+| runtime | 22 s probing + comparing (0 s fetch/build before it) |
+
+## Results
+
+A file's class is the first that applies:
+
+- **panic / hang / crash / oom** — clawhdf5 panicked (caught per object or not), hit the timeout, died on a signal, or failed an allocation. The CI gate fails on any of these.
+- **h5py-cannot-read** — libhdf5 could not open the file (or itself crashed or hung). Nothing to compare against; most are the deliberately malformed CVE reproducers.
+- **our-error** — clawhdf5 returned an error for something h5py reads.
+- **mismatch** — both read it, but the shapes, values, object set or attribute set differ.
+- **ok** — every object h5py reads, clawhdf5 reads identically.
+
+| corpus | files | ok | our-error | mismatch | h5py-cannot-read | panic | hang | crash | oom |
+|---|---|---|---|---|---|---|---|---|---|
+| NCAS-CMS_pyfive | 33 | 32 | 0 | 1 | 0 | 0 | 0 | 0 | 0 |
+| cve_hdf5 | 147 | 100 | 6 | 9 | 32 | 0 | 0 | 0 | 0 |
+| h5py_data | 4 | 4 | 0 | 0 | 0 | 0 | 0 | 0 | 0 |
+| hdf5 | 466 | 392 | 4 | 10 | 60 | 0 | 0 | 0 | 0 |
+| netcdf-c | 20 | 20 | 0 | 0 | 0 | 0 | 0 | 0 | 0 |
+| netcdf4-python | 18 | 18 | 0 | 0 | 0 | 0 | 0 | 0 | 0 |
+| usnistgov_h5wasm | 5 | 5 | 0 | 0 | 0 | 0 | 0 | 0 | 0 |
+| xarray-data | 4 | 4 | 0 | 0 | 0 | 0 | 0 | 0 | 0 |
+| **all** | **697** | **575** | **10** | **20** | **92** | **0** | **0** | **0** | **0** |
+
+2 of the 20 mismatches are a known h5py bug, not ours (see *Known not-our-bug*).
+
+Corpora (fetched by `conformance/fetch-corpus.sh` into the gitignored `conformance/.cache/`):
+
+| corpus | source | commit |
+|---|---|---|
+| hdf5 | https://github.com/HDFGroup/hdf5 | `a3cf1ea82cc7` |
+| cve_hdf5 | https://github.com/HDFGroup/cve_hdf5 | `3fd1f5ae3869` |
+| netcdf-c | https://github.com/Unidata/netcdf-c | `beb7b9585273` |
+| NCAS-CMS_pyfive | https://github.com/NCAS-CMS/pyfive | `8cf07b874913` |
+| usnistgov_h5wasm | https://github.com/usnistgov/h5wasm | `02f6336527d2` |
+| netcdf4-python | https://github.com/Unidata/netcdf4-python | `6e67576d39ae` |
+| xarray-data | https://github.com/pydata/xarray-data | `a35297e9da2c` |
+| h5py_data | https://github.com/h5py/h5py (`h5py/tests/data_files`) | `b2f0347c4200` |
+
+## Panics, hangs, crashes, out-of-memory
+
+None.
+
+## Our-error root causes
+
+Grouped by normalised error message. *files* counts files whose class this cause affects.
+
+| files | objects | error | examples |
+|---:|---:|---|---|
+| 3 | 3 | `DataSizeMismatch { expected: N, actual: N }` | `cve_hdf5/cvefiles/cve-2020-18494.h5`, `cve_hdf5/cvefiles/cve-2024-32623.h5`, `cve_hdf5/cvefiles/cve-2025-2309.h5` |
+| 2 | 2 | `ChunkedReadError("…")` | `cve_hdf5/cvefiles/cve-2025-2308.h5`, `hdf5/test/testfiles/bad_nbit_parms_walk.h5` |
+| 2 | 2 | `UnsupportedFilter(N)` | `hdf5/HDF5Examples/C/H5FLT/tfiles/h5ex_d_blosc2.h5`, `hdf5/HDF5Examples/C/H5FLT/tfiles/h5ex_d_zfp.h5` |
+| 1 | 1 | `UnexpectedEof { expected: N, available: N }` | `cve_hdf5/cvefiles/cve-2019-9151.h5` |
+| 1 | 1 | `MissingMessage(Dataspace)` | `cve_hdf5/cvefiles/cve-2024-33874.h5` |
+| 1 | 1 | `InvalidObjectHeaderVersion(N)` | `hdf5/tools/test/testfiles/h5clear_mdc_image.h5` |
+
+## Mismatch root causes
+
+| files | objects | cause | examples |
+|---:|---:|---|---|
+| 13 | 14 | `missing-object` | `cve_hdf5/cvefiles/cve-2019-8397.h5`, `cve_hdf5/cvefiles/cve-2019-8398.h5`, `cve_hdf5/cvefiles/cve-2021-46243.h5` (+10 more) |
+| 2 | 6 | `extra-attr` | `cve_hdf5/cvefiles/cve-2018-17438`, `cve_hdf5/cvefiles/cve-2018-17439` |
+| 1 | 1 | `attr-values: ours=vlen(>u8) h5py=object layout=- filters=-` | `NCAS-CMS_pyfive/tests/data/attr_datatypes.hdf5` |
+| 1 | 4 | `extra-object` | `cve_hdf5/cvefiles/cve-2021-46244.h5` |
+| 1 | 1 | `values: ours=<f4 h5py=float32 layout=chunked filters=-` | `cve_hdf5/cvefiles/cve-2025-44904.h5` |
+| 1 | 1 | `values: ours=>i2 h5py=>i2 layout=chunked filters=[6]` | `cve_hdf5/cvefiles/cve-2025-44905.h5` |
+| 1 | 1 | `values: ours=>f4 h5py=>f4 layout=chunked filters=[2]` | `cve_hdf5/cvefiles/cve-2025-44905.h5` |
+| 1 | 1 | `values: ours=<f4 h5py=float32 layout=chunked filters=[2]` | `cve_hdf5/cvefiles/cve-2025-44905.h5` |
+| 1 | 1 | `values: ours=((<i4)[6, 3])[4] h5py=(('<i4', (6, 3)), (4,)) layout=contiguous filters=-` | `hdf5/tools/test/testfiles/tarray3.h5` |
+| 1 | 1 | `values: ours=vlen({r:>f4,i:>f4}8) h5py=object layout=contiguous filters=-` | `hdf5/tools/test/testfiles/tcomplex_be.h5` |
+
+## CVE corpus: clawhdf5 vs h5dump vs h5py
+
+The 147 files of [HDFGroup/cve_hdf5](https://github.com/HDFGroup/cve_hdf5) — reproducers for
+published libhdf5 CVEs and fuzzer finds. *read* = produced output (possibly with per-object
+errors), *error* = refused cleanly. h5dump exits non-zero on any error anywhere in a file, so
+its read/error split is not comparable with the other two rows; the panic, crash, hang and oom
+columns are.
+
+| tool | read | error | panic | crash | hang | oom |
+|---|---:|---:|---:|---:|---:|---:|
+| clawhdf5 | 140 | 7 | 0 | 0 | 0 | 0 |
+| h5dump 1.14.6 | 16 | 129 | 0 | 2 | 0 | 0 |
+| h5py 3.16.0 / HDF5 2.0.0 | 115 | 31 | 0 | 1 | 0 | 0 |
+
+<details><summary>Per-file outcomes</summary>
+
+| file | h5dump | h5py | clawhdf5 | class |
+|---|---|---|---|---|
+| cvefiles/cve-2016-4330.h5 | error exit | read 3 obj, 1 errors | read 3 obj, 1 errors | ok |
+| cvefiles/cve-2016-4331.h5 | error exit | read 25 obj, 1 errors | read 25 obj, 1 errors | ok |
+| cvefiles/cve-2016-4332-mtime-new.h5 | error exit | read 25 obj, 1 errors | read 25 obj, 1 errors | ok |
+| cvefiles/cve-2016-4332-mtime.h5 | error exit | read 4 obj, 3 errors | read 4 obj, 3 errors | ok |
+| cvefiles/cve-2016-4332-stab.h5 | error exit | open error | read 1 obj, 1 errors | h5py-cannot-read |
+| cvefiles/cve-2016-4333.h5 | error exit | read 3 obj, 1 errors | read 3 obj, 1 errors | ok |
+| cvefiles/cve-2017-17505.h5 | error exit | read 2 obj, 1 errors | read 2 obj, 1 errors | ok |
+| cvefiles/cve-2017-17506.h5 | error exit | read 2 obj, 1 errors | read 2 obj, 1 errors | ok |
+| cvefiles/cve-2017-17507.h5 | error exit | read 2 obj, 1 errors | read 2 obj, 1 errors | ok |
+| cvefiles/cve-2017-17508.h5 | error exit | read 2 obj, 1 errors | read 2 obj, 1 errors | ok |
+| cvefiles/cve-2017-17509.h5 | error exit | read 2 obj, 1 errors | read 2 obj, 1 errors | ok |
+| cvefiles/cve-2018-11202.h5 | error exit | read 2 obj, 1 errors | read 2 obj, 1 errors | ok |
+| cvefiles/cve-2018-11203.h5 | error exit | read 2 obj, 1 errors | read 2 obj, 1 errors | ok |
+| cvefiles/cve-2018-11204.h5 | error exit | read 2 obj, 1 errors | read 2 obj, 1 errors | ok |
+| cvefiles/cve-2018-11205.h5 | error exit | read 2 obj, 1 errors | read 2 obj, 1 errors | ok |
+| cvefiles/cve-2018-11206-new.h5 | error exit | read 3 obj, 1 errors | read 3 obj, 1 errors | ok |
+| cvefiles/cve-2018-11206-old.h5 | error exit | read 3 obj, 1 errors | read 3 obj, 1 errors | ok |
+| cvefiles/cve-2018-11207.h5 | error exit | read 2 obj, 1 errors | read 2 obj, 1 errors | ok |
+| cvefiles/cve-2018-13866.h5 | error exit | open error | open error | h5py-cannot-read |
+| cvefiles/cve-2018-13867.h5 | error exit | read 1 obj, 1 errors | read 1 obj, 1 errors | ok |
+| cvefiles/cve-2018-13868.h5 | error exit | read 3 obj, 1 errors | read 3 obj, 1 errors | ok |
+| cvefiles/cve-2018-13869.h5 | error exit | read 1 obj, 1 errors | read 1 obj, 1 errors | ok |
+| cvefiles/cve-2018-13870.h5 | error exit | read 1 obj, 1 errors | read 1 obj, 1 errors | ok |
+| cvefiles/cve-2018-13871.h5 | error exit | read 2 obj | read 2 obj | ok |
+| cvefiles/cve-2018-13872.h5 | error exit | read 1 obj, 1 errors | read 1 obj, 1 errors | ok |
+| cvefiles/cve-2018-13873.h5 | error exit | read 1 obj, 1 errors | read 1 obj, 1 errors | ok |
+| cvefiles/cve-2018-13874.h5 | error exit | open error | open error | h5py-cannot-read |
+| cvefiles/cve-2018-13875.h5 | error exit | read 3 obj, 1 errors | read 3 obj, 1 errors | ok |
+| cvefiles/cve-2018-13876.h5 | error exit | open error | open error | h5py-cannot-read |
+| cvefiles/cve-2018-14031.h5 | error exit | read 3 obj, 1 errors | read 3 obj, 1 errors | ok |
+| cvefiles/cve-2018-14033.h5 | error exit | read 3 obj, 1 errors | read 3 obj, 1 errors | ok |
+| cvefiles/cve-2018-14034.h5 | error exit | read 1 obj, 2 errors | read 1 obj | ok |
+| cvefiles/cve-2018-14035.h5 | error exit | read 3 obj, 1 errors | read 3 obj, 1 errors | ok |
+| cvefiles/cve-2018-14460.h5 | error exit | read 3 obj, 2 errors | read 3 obj, 2 errors | ok |
+| cvefiles/cve-2018-15671.h5 | ok | read 1 obj | read 1 obj | ok |
+| cvefiles/cve-2018-15672.h5 | error exit | read 2 obj, 1 errors | read 2 obj, 1 errors | ok |
+| cvefiles/cve-2018-16438.h5 | error exit | read 1 obj, 1 errors | read 1 obj | ok |
+| cvefiles/cve-2018-17233.h5 | error exit | read 6 obj, 1 errors | read 6 obj, 1 errors | ok |
+| cvefiles/cve-2018-17234.h5 | error exit | read 6 obj, 1 errors | read 6 obj, 1 errors | ok |
+| cvefiles/cve-2018-17237.h5 | error exit | read 2 obj, 1 errors | read 2 obj, 1 errors | ok |
+| cvefiles/cve-2018-17432.h5 | error exit | read 3 obj, 1 errors | read 3 obj, 1 errors | ok |
+| cvefiles/cve-2018-17433 | error exit | open error | open error | h5py-cannot-read |
+| cvefiles/cve-2018-17434.h5 | error exit | read 3 obj, 1 errors | read 3 obj, 1 errors | ok |
+| cvefiles/cve-2018-17435.h5 | error exit | read 3 obj, 1 errors | read 3 obj, 1 errors | ok |
+| cvefiles/cve-2018-17436 | error exit | open error | open error | h5py-cannot-read |
+| cvefiles/cve-2018-17437.h5 | error exit | read 3 obj, 1 errors | read 3 obj, 1 errors | ok |
+| cvefiles/cve-2018-17438 | error exit | read 3 obj, 1 errors | read 3 obj, 1 errors | mismatch |
+| cvefiles/cve-2018-17439 | error exit | read 3 obj, 1 errors | read 3 obj, 1 errors | mismatch |
+| cvefiles/cve-2019-8396.h5 | error exit | read 3 obj, 2 errors | read 3 obj, 2 errors | ok |
+| cvefiles/cve-2019-8397.h5 | error exit | read 3 obj, 2 errors | read 2 obj, 1 errors | mismatch |
+| cvefiles/cve-2019-8398.h5 | error exit | read 3 obj, 2 errors | read 2 obj, 1 errors | mismatch |
+| cvefiles/cve-2019-9151.h5 | error exit | read 3 obj, 1 errors | read 3 obj, 2 errors | our-error |
+| cvefiles/cve-2019-9152.h5 | error exit | read 3 obj, 1 errors | read 3 obj, 1 errors | ok |
+| cvefiles/cve-2020-10809 | error exit | open error | open error | h5py-cannot-read |
+| cvefiles/cve-2020-10810.h5 | error exit | open error | read 2 obj | h5py-cannot-read |
+| cvefiles/cve-2020-10811.h5 | error exit | read 25 obj, 1 errors | read 25 obj, 1 errors | ok |
+| cvefiles/cve-2020-10812.h5 | error exit | open error | read 2 obj | h5py-cannot-read |
+| cvefiles/cve-2020-18232.h5 | error exit | read 3 obj, 2 errors | read 3 obj, 2 errors | ok |
+| cvefiles/cve-2020-18494.h5 | ok | read 2 obj | read 2 obj, 1 errors | our-error |
+| cvefiles/cve-2021-36977.h5 | error exit | read 1 obj, 1 errors | read 1 obj, 1 errors | ok |
+| cvefiles/cve-2021-37501.h5 | error exit | read 18 obj, 1 errors | read 18 obj, 1 errors | ok |
+| cvefiles/cve-2021-45829.h5 | error exit | read 1 obj, 2 errors | read 1 obj, 1 errors | ok |
+| cvefiles/cve-2021-45830.h5 | error exit | open error | read 1 obj, 1 errors | h5py-cannot-read |
+| cvefiles/cve-2021-45833.h5 | error exit | read 2 obj, 1 errors | read 2 obj, 1 errors | ok |
+| cvefiles/cve-2021-46242.h5 | error exit | open error | read 1 obj, 1 errors | h5py-cannot-read |
+| cvefiles/cve-2021-46243.h5 | error exit | read 3 obj, 2 errors | read 2 obj, 1 errors | mismatch |
+| cvefiles/cve-2021-46244.h5 | error exit | read 2 obj, 1 errors | read 6 obj, 4 errors | mismatch |
+| cvefiles/cve-2024-29157.h5 | error exit | read 4 obj, 7 errors | read 4 obj, 7 errors | ok |
+| cvefiles/cve-2024-29158.h5 | ok | read 3 obj, 1 errors | read 3 obj, 1 errors | ok |
+| cvefiles/cve-2024-29159.h5 | error exit | read 2 obj, 1 errors | read 2 obj, 1 errors | ok |
+| cvefiles/cve-2024-29160.h5 | error exit | read 4 obj, 1 errors | read 4 obj, 1 errors | ok |
+| cvefiles/cve-2024-29161.h5 | error exit | read 1 obj, 1 errors | read 1 obj, 1 errors | ok |
+| cvefiles/cve-2024-29162.h5 | error exit | read 17 obj, 4 errors | read 17 obj, 4 errors | ok |
+| cvefiles/cve-2024-29163.h5 | error exit | read 7 obj, 1 errors | read 7 obj, 1 errors | ok |
+| cvefiles/cve-2024-29164.h5 | ok | read 3 obj | read 3 obj | ok |
+| cvefiles/cve-2024-29165.h5 | error exit | read 2 obj, 1 errors | read 2 obj, 1 errors | ok |
+| cvefiles/cve-2024-29166.h5 | error exit | read 17 obj, 2 errors | read 17 obj | ok |
+| cvefiles/cve-2024-32605.h5 | ok | read 6 obj, 1 errors | read 6 obj, 1 errors | ok |
+| cvefiles/cve-2024-32606.h5 | error exit | read 2 obj, 1 errors | read 2 obj, 1 errors | ok |
+| cvefiles/cve-2024-32607-1.h5 | ok | read 10 obj | read 10 obj | ok |
+| cvefiles/cve-2024-32607-2.h5 | error exit | read 9 obj, 1 errors | read 9 obj, 1 errors | ok |
+| cvefiles/cve-2024-32608.h5 | error exit | read 6 obj, 1 errors | read 6 obj | ok |
+| cvefiles/cve-2024-32609.h5 | error exit | SIGSEGV | read 3 obj, 1 errors | h5py-cannot-read |
+| cvefiles/cve-2024-32610.h5 | error exit | read 2 obj, 1 errors | read 2 obj, 1 errors | ok |
+| cvefiles/cve-2024-32611.h5 | ok | read 6 obj | read 6 obj | ok |
+| cvefiles/cve-2024-32612.h5 | ok | read 3 obj | read 3 obj | ok |
+| cvefiles/cve-2024-32613.h5 | error exit | read 7 obj, 1 errors | read 7 obj, 1 errors | ok |
+| cvefiles/cve-2024-32614.h5 | error exit | read 25 obj, 2 errors | read 25 obj, 2 errors | ok |
+| cvefiles/cve-2024-32615.h5 | error exit | read 4 obj, 1 errors | read 4 obj, 1 errors | ok |
+| cvefiles/cve-2024-32616.h5 | error exit | read 10 obj, 7 errors | read 10 obj, 6 errors | ok |
+| cvefiles/cve-2024-32617.h5 | error exit | read 1 obj, 1 errors | read 1 obj, 1 errors | ok |
+| cvefiles/cve-2024-32618.h5 | error exit | read 4 obj, 2 errors | read 3 obj, 1 errors | mismatch |
+| cvefiles/cve-2024-32619.h5 | error exit | read 3 obj, 2 errors | read 3 obj, 2 errors | ok |
+| cvefiles/cve-2024-32620.h5 | error exit | read 1 obj, 1 errors | read 1 obj, 1 errors | ok |
+| cvefiles/cve-2024-32621.h5 | ok | read 3 obj, 1 errors | read 3 obj, 1 errors | ok |
+| cvefiles/cve-2024-32622.h5 | ok | read 3 obj, 1 errors | read 3 obj, 1 errors | ok |
+| cvefiles/cve-2024-32623.h5 | ok | read 6 obj | read 6 obj, 1 errors | our-error |
+| cvefiles/cve-2024-32624.h5 | error exit | read 6 obj, 1 errors | read 6 obj | ok |
+| cvefiles/cve-2024-33873.h5 | error exit | read 4 obj, 1 errors | read 4 obj, 1 errors | ok |
+| cvefiles/cve-2024-33874.h5 | ok | read 6 obj, 1 errors | read 6 obj, 2 errors | our-error |
+| cvefiles/cve-2024-33875.h5 | ok | read 2 obj | read 2 obj | ok |
+| cvefiles/cve-2024-33876.h5 | ok | read 3 obj, 1 errors | read 3 obj, 1 errors | ok |
+| cvefiles/cve-2024-33877.h5 | error exit | read 8 obj, 1 errors | read 8 obj, 1 errors | ok |
+| cvefiles/cve-2025-2153.h5 | error exit | open error | read 1 obj, 1 errors | h5py-cannot-read |
+| cvefiles/cve-2025-2308.h5 | error exit | read 25 obj, 1 errors | read 25 obj, 2 errors | our-error |
+| cvefiles/cve-2025-2309.h5 | ok | read 6 obj, 1 errors | read 6 obj, 1 errors | our-error |
+| cvefiles/cve-2025-2310.h5 | error exit | read 24 obj, 8 errors | read 24 obj, 8 errors | ok |
+| cvefiles/cve-2025-2912.h5 | error exit | open error | read 1 obj, 1 errors | h5py-cannot-read |
+| cvefiles/cve-2025-2913.h5 | error exit | open error | read 1 obj, 1 errors | h5py-cannot-read |
+| cvefiles/cve-2025-2914.h5 | error exit | open error | read 1 obj, 1 errors | h5py-cannot-read |
+| cvefiles/cve-2025-2915.h5 | error exit | open error | read 1 obj, 1 errors | h5py-cannot-read |
+| cvefiles/cve-2025-2923.h5 | error exit | open error | read 1 obj, 1 errors | h5py-cannot-read |
+| cvefiles/cve-2025-2924.h5 | error exit | read 1 obj, 1 errors | read 1 obj, 1 errors | ok |
+| cvefiles/cve-2025-2925.h5 | error exit | read 1 obj, 1 errors | read 1 obj, 1 errors | ok |
+| cvefiles/cve-2025-2926.h5 | error exit | open error | read 1 obj, 1 errors | h5py-cannot-read |
+| cvefiles/cve-2025-44904.h5 | error exit | read 25 obj, 1 errors | read 25 obj, 1 errors | mismatch |
+| cvefiles/cve-2025-44905.h5 | error exit | read 25 obj, 3 errors | read 25 obj, 3 errors | mismatch |
+| cvefiles/cve-2025-6269-1.h5 | error exit | read 1 obj, 1 errors | read 1 obj, 1 errors | ok |
+| cvefiles/cve-2025-6269-2.h5 | error exit | read 1 obj, 1 errors | read 1 obj, 1 errors | ok |
+| cvefiles/cve-2025-6269-3.h5 | error exit | read 1 obj, 1 errors | read 1 obj, 1 errors | ok |
+| cvefiles/cve-2025-6269-4.h5 | error exit | read 1 obj, 1 errors | read 1 obj, 1 errors | ok |
+| cvefiles/cve-2025-6270-1.h5 | error exit | open error | read 1 obj, 1 errors | h5py-cannot-read |
+| cvefiles/cve-2025-6270-2.h5 | error exit | open error | read 1 obj, 1 errors | h5py-cannot-read |
+| cvefiles/cve-2025-6270-3.h5 | error exit | open error | read 1 obj, 1 errors | h5py-cannot-read |
+| cvefiles/cve-2025-6516.h5 | error exit | read 1 obj, 1 errors | read 1 obj, 1 errors | ok |
+| cvefiles/cve-2025-6750.h5 | error exit | open error | read 1 obj, 1 errors | h5py-cannot-read |
+| cvefiles/cve-2025-6816.h5 | error exit | open error | read 1 obj, 1 errors | h5py-cannot-read |
+| cvefiles/cve-2025-6817.h5 | error exit | open error | read 1 obj | h5py-cannot-read |
+| cvefiles/cve-2025-6818.h5 | error exit | open error | read 1 obj | h5py-cannot-read |
+| cvefiles/cve-2025-6856.h5 | error exit | open error | read 1 obj, 1 errors | h5py-cannot-read |
+| cvefiles/cve-2025-6857.h5 | error exit | read 1 obj, 1 errors | read 1 obj, 1 errors | ok |
+| cvefiles/cve-2025-6858.h5 | SIGSEGV | open error | read 1 obj, 1 errors | h5py-cannot-read |
+| cvefiles/cve-2025-7067.h5 | error exit | read 1 obj, 1 errors | read 1 obj, 1 errors | ok |
+| cvefiles/cve-2025-7068.h5 | error exit | open error | read 1 obj, 1 errors | h5py-cannot-read |
+| cvefiles/cve-2025-7069.h5 | error exit | open error | read 1 obj, 1 errors | h5py-cannot-read |
+| cvefiles/cve-2026-26200.h5 | error exit | read 1 obj, 1 errors | read 1 obj, 1 errors | ok |
+| cvefiles/cve-2026-34734.h5 | error exit | read 2 obj, 1 errors | read 2 obj, 1 errors | ok |
+| cvefiles/cve-2026-92627.h5 | error exit | read 2 obj, 1 errors | read 2 obj, 1 errors | ok |
+| cvefiles/unknown-1.h5 | error exit | read 11 obj, 1 errors | read 11 obj, 1 errors | ok |
+| fuzzerfiles/gh-4431-poc-03.h5 | error exit | read 1 obj | read 1 obj | ok |
+| fuzzerfiles/gh-4432-poc-05.h5 | SIGSEGV | read 1 obj, 1 errors | read 1 obj, 1 errors | ok |
+| fuzzerfiles/gh-4433-poc-08.h5 | error exit | read 1 obj, 1 errors | read 1 obj, 1 errors | ok |
+| fuzzerfiles/gh-4434-poc-09.h5 | error exit | open error | read 1 obj, 1 errors | h5py-cannot-read |
+| fuzzerfiles/gh-4435-poc-10.h5 | error exit | read 1 obj, 1 errors | read 1 obj, 1 errors | ok |
+| fuzzerfiles/gh-4585.h5 | error exit | open error | open error | h5py-cannot-read |
+| fuzzerfiles/gh_2649_flawed.h5 | error exit | read 9 obj, 1 errors | read 9 obj, 1 errors | ok |
+| fuzzerfiles/gh_2649_plain_model.h5 | ok | read 10 obj | read 10 obj | ok |
+
+</details>
+
+## Known not-our-bug
+
+- **h5py big-endian variable-length sequences.** h5py returns the elements of a VL sequence
+  whose base type is big-endian with the file's big-endian bytes but a native (little-endian)
+  numpy dtype, so the values it reports are byte-swapped garbage; `h5dump` prints the values
+  clawhdf5 reads. Reproducer: `h5py.vlen_dtype(np.dtype('>f4'))` dataset holding `[1.0, 2.0]`
+  reads back in h5py as `[4.6e-41, 9.0e-44]`. Affected here: `NCAS-CMS_pyfive/tests/data/attr_datatypes.hdf5`, `hdf5/tools/test/testfiles/tcomplex_be.h5`.
+- **Non-IEEE floats and partial-precision integers (N-Bit).** libhdf5 converts a float whose
+  bit layout is not IEEE (e.g. `H5Tset_precision` for the N-Bit filter) or an integer with a
+  bit offset / reduced precision into the plain numpy type of the same size. The probe
+  compares such values as converted numbers, not raw file bytes (before 2026-09-25 it compared
+  raw bytes, which reported every N-Bit float dataset as a mismatch).
+- **Types h5py widens.** Where h5py reads a type into a numpy type of a different size
+  (FP8 -> float16, bfloat16 -> float32, x87 long double -> float128) the values are not
+  compared (shape and presence still are): dataset file type size 1 -> numpy float16 (2) (15x), attr file type size 1 -> numpy float16 (2) (15x), dataset file type size 2 -> numpy float32 (4) (2x), dataset file type size 8 -> numpy float128 (16) (1x), dataset file type size 12 -> numpy float128 (16) (1x), attr file type size 2 -> numpy float32 (4) (1x), dataset file type size 2 -> numpy >f4 (4) (1x), attr file type size 2 -> numpy >f4 (4) (1x).
+- **References** are compared by presence only (`R`), not by target.
+
+## Objects h5py fails on but clawhdf5 reads
+
+- 19 x `OSError: Can't synchronously read data (no appropriate function for conversion path)`
+- 1 x `TypeError: unhandled dtype kind M (dtype('…'))`
+- 1 x `TypeError: No NumPy equivalent for TypeTimeID exists`
+- 1 x `KeyError: "…"`
+- 1 x `ValueError: Insufficient precision in available types to represent (N, N, N, N, N)`
+
+## Reproduce
+
+```sh
+# needs: Rust, python3 with h5py numpy hdf5plugin (conformance/requirements.txt), h5dump (hdf5-tools), git
+CLAWHDF5_PYTHON=/path/to/venv/bin/python conformance/run.sh
+```
+
+The corpus (about 450 MB of sparse checkouts) is cached in `conformance/.cache/`; results for
+every file, both sides' raw JSON and stderr, are in `conformance/.cache/results/`.
+`conformance/baseline.json` holds the ok files the nightly CI job (`.gitea/workflows/conformance.yml`)
+must keep; `conformance/run.sh --update-baseline` rewrites it.

Cargo.toml

@@ -16,6 +16,8 @@ members = [
     "crates/clawhdf5-cli",
     "crates/clawhdf5-napi",
     "crates/clawhdf5-bench",
+    "crates/clawhdf5-tools",
+    "crates/clawhdf5-wasm",
     "crates/libaec-sys",
 ]
 resolver = "2"
@@ -34,3 +36,12 @@ tempfile = "3"
 criterion = { version = "0.5", features = ["html_reports"] }
 half = "2.7"
 serde = { version = "1", features = ["derive"] }
+
+# The browser build of clawhdf5-wasm (examples/wasm-viewer/build.sh): size
+# over speed, whole-program optimisation. Native profiles are unaffected.
+[profile.wasm-release]
+inherits = "release"
+opt-level = "s"
+lto = true
+codegen-units = 1
+panic = "abort"

README.md

@@ -587,14 +587,14 @@ let exported = backend.export_markdown("MEMORY.md")?;
 ## Crate Map
 
 ```
-clawhdf5 workspace (16 crates, ~86K lines of Rust in src/, ~104K with tests
+clawhdf5 workspace (17 crates, ~86K lines of Rust in src/, ~104K with tests
                      and benches; plus libaec-sys, an internal FFI bindings
                      crate for the optional szip feature)
 │
 ├── Core HDF5
 │   ├── clawhdf5-format      — Binary parser/writer (no_std-capable), shared type definitions
 │   ├── clawhdf5-io          — I/O abstraction (file/memory readers; optional mmap, async, HSDS, MPI)
-│   ├── clawhdf5-filters     — Fast deflate path (zlib-ng); lz4/zstd/pcodec/szip filters live in clawhdf5-format
+│   ├── clawhdf5-filters     — Fast deflate path (zlib-ng); the filter registry and the lz4/zstd/pcodec/szip/LZF/bitshuffle/bzip2/Blosc filters live in clawhdf5-format
 │   ├── clawhdf5-derive      — Proc macros
 │   ├── clawhdf5             — High-level API
 │   ├── clawhdf5-netcdf4     — NetCDF-4 support
@@ -610,7 +610,8 @@ clawhdf5 workspace (16 crates, ~86K lines of Rust in src/, ~104K with tests
 │
 ├── Bindings
 │   ├── clawhdf5-py          — Python (PyO3)
-│   └── clawhdf5-napi        — Node.js (napi-rs)
+│   ├── clawhdf5-napi        — Node.js (napi-rs)
+│   └── clawhdf5-wasm        — Browser (WebAssembly, wasm-bindgen; read-only)
 │
 └── Tooling
     └── clawhdf5-bench       — Benchmark suite
@@ -700,6 +701,22 @@ stores keep their setting. Opt out with `float16 = false` or
 | `system-zlib` | no | System zlib backend for deflate (C) |
 | `blake3_hash` | no | BLAKE3 content hashing for provenance |
 | `szip` | no | SZIP filter (id 4) via libaec (C, through the internal `libaec-sys` crate) |
+| `lzf` | **yes** | LZF filter (id 32000), h5py's built-in `compression="lzf"`: read and write. No dependencies |
+| `bitshuffle` | no | Bitshuffle filter (id 32008) with its LZ4 and Zstandard modes: read and write. Pure Rust (lz4_flex, ruzstd) |
+| `bzip2` | no | bzip2 filter (id 307): read and write. Pure Rust (the `bzip2` crate's libbz2-rs-sys backend compiles no C) |
+| `blosc` | no | Blosc 1 filter (id 32001): reads BloscLZ, LZ4/LZ4HC, Snappy, Zlib and Zstandard frames with byte or bit shuffle; writes LZ4, Snappy, Zlib or Zstandard (not BloscLZ). Pure Rust |
+| `plugin-filters` | no | All four above |
+
+Blosc2 (32026) and ZFP (32013) are not implemented: reading them fails with
+`UnsupportedFilter`, whose message names the filter. Any other filter can be
+supplied at run time with `filter_registry::register_filter` (a decoder
+closure, or a `FilterCodec` that also encodes). The facade (`clawhdf5`)
+forwards `lzf`, `bitshuffle`, `bzip2`, `blosc` and `plugin-filters`. Write
+with `DatasetBuilder::with_lzf()`, `with_bitshuffle(..)`, `with_bzip2(..)`
+and `with_blosc(..)`; h5py + hdf5plugin read the result (tested both ways in
+`crates/clawhdf5/tests/plugin_filters_interop.rs`). The pure-Rust Zstandard
+encoder has one level (about zstd's level 1); no speed or ratio claims are
+made for these codecs.
 
 ### `clawhdf5-ann`
 

conformance/.gitignore

@@ -0,0 +1,3 @@
+/.cache/
+# pin the probe's dependencies (the workspace lock is not committed)
+!/probe/Cargo.lock

conformance/README.md

@@ -0,0 +1,39 @@
+# Conformance sweep
+
+Reads every HDF5 file of eight public corpora with clawhdf5 and with
+h5py/libhdf5, compares the two readings object by object, and writes
+[`CONFORMANCE.md`](../CONFORMANCE.md).
+
+```sh
+CLAWHDF5_PYTHON=/path/to/venv/bin/python conformance/run.sh      # ~30 s once the corpus is cached
+conformance/run.sh --update-baseline                             # after an intended change in results
+```
+
+Needs Rust, `git`, `h5dump` (Debian/Ubuntu `hdf5-tools`), `libaec` (for the
+probe's `szip` feature; `libaec-dev`), and a Python with the packages in
+`requirements.txt`. The first run downloads about 450 MB of sparse checkouts.
+
+| file | role |
+|---|---|
+| `corpus.txt` | the corpora: git URL, pinned commit, swept root, sparse-checkout patterns |
+| `fetch-corpus.sh` | shallow, sparse, blob-filtered checkout of each pinned commit into `.cache/src/` (gitignored); no-op when already there |
+| `list_files.py` | which files are probed (HDF5/netCDF-4 extensions minus netCDF classic, plus the CVE reproducers) |
+| `probe/` | the clawhdf5 side: a standalone crate (outside the workspace, so `cargo test --workspace` never builds it) that walks a file with `clawhdf5-format` and prints canonical JSON |
+| `ref.py` | the h5py side: the same JSON from h5py |
+| `run_one.sh` | runs both sides on one file (and `h5dump` on the CVE corpus) under a timeout and an address-space limit |
+| `compare.py` | classifies each file (ok / our-error / mismatch / h5py-cannot-read / panic / hang / crash / oom) and groups root causes |
+| `report.py` | writes `CONFORMANCE.md` |
+| `check.py` | the gate: fails on any panic/hang/crash/oom, on an ok count below `baseline.json`, or on a baseline-ok file that is no longer ok |
+| `baseline.json` | the ok files the gate holds the line on |
+| `requirements.txt` | pinned h5py / numpy / hdf5plugin / netCDF4 |
+
+Results for every file (both sides' JSON and stderr, `results.csv`,
+`results.json`, `summary.md`) are left in `.cache/results/`.
+
+The nightly job is `.gitea/workflows/conformance.yml`; it prints the report
+into the job log.
+
+The canonical value encoding both sides hash is documented at the top of
+`probe/src/main.rs`. Values are compared as libhdf5 presents them: a float
+with a non-IEEE bit layout (N-Bit) or an integer with a bit offset is compared
+as the converted number, not as raw file bytes.

conformance/baseline.json

@@ -0,0 +1,624 @@
+{
+ "comment": "conformance/run.sh fails if the ok count drops below `ok` or a file in `ok_files` stops being ok. Regenerate with `conformance/run.sh --update-baseline` after an intended change.",
+ "commit": "72306c601399748616bc9d061be2ebc4c1bea9e0",
+ "date": "2026-09-26 06:50 UTC",
+ "reference": "h5py 3.16.0 / HDF5 2.0.0",
+ "files": 697,
+ "ok": 575,
+ "counts": {
+  "h5py-cannot-read": 92,
+  "mismatch": 20,
+  "ok": 575,
+  "our-error": 10
+ },
+ "per_corpus": {
+  "NCAS-CMS_pyfive": {
+   "mismatch": 1,
+   "ok": 32
+  },
+  "cve_hdf5": {
+   "h5py-cannot-read": 32,
+   "mismatch": 9,
+   "ok": 100,
+   "our-error": 6
+  },
+  "h5py_data": {
+   "ok": 4
+  },
+  "hdf5": {
+   "h5py-cannot-read": 60,
+   "mismatch": 10,
+   "ok": 392,
+   "our-error": 4
+  },
+  "netcdf-c": {
+   "ok": 20
+  },
+  "netcdf4-python": {
+   "ok": 18
+  },
+  "usnistgov_h5wasm": {
+   "ok": 5
+  },
+  "xarray-data": {
+   "ok": 4
+  }
+ },
+ "ok_files": [
+  "NCAS-CMS_pyfive/tests/compact.hdf5",
+  "NCAS-CMS_pyfive/tests/data/btreev2.hdf5",
+  "NCAS-CMS_pyfive/tests/data/chunked.hdf5",
+  "NCAS-CMS_pyfive/tests/data/cmip_bad_eg.nc",
+  "NCAS-CMS_pyfive/tests/data/compressed.hdf5",
+  "NCAS-CMS_pyfive/tests/data/compressed_v1.hdf5",
+  "NCAS-CMS_pyfive/tests/data/dataset_datatypes.hdf5",
+  "NCAS-CMS_pyfive/tests/data/dataset_multidim.hdf5",
+  "NCAS-CMS_pyfive/tests/data/dim_scales.hdf5",
+  "NCAS-CMS_pyfive/tests/data/earliest.hdf5",
+  "NCAS-CMS_pyfive/tests/data/enum_h5variable.hdf5",
+  "NCAS-CMS_pyfive/tests/data/enum_variable.hdf5",
+  "NCAS-CMS_pyfive/tests/data/enum_variable.nc",
+  "NCAS-CMS_pyfive/tests/data/enums_from_netcdf.nc",
+  "NCAS-CMS_pyfive/tests/data/fillvalue_earliest.hdf5",
+  "NCAS-CMS_pyfive/tests/data/fillvalue_latest.hdf5",
+  "NCAS-CMS_pyfive/tests/data/filter_pipeline_v2.hdf5",
+  "NCAS-CMS_pyfive/tests/data/fletcher32.hdf5",
+  "NCAS-CMS_pyfive/tests/data/fractal_heap_no_mci_rlat.nc",
+  "NCAS-CMS_pyfive/tests/data/groups.hdf5",
+  "NCAS-CMS_pyfive/tests/data/h5netcdf_test.hdf5",
+  "NCAS-CMS_pyfive/tests/data/issue23_A.nc",
+  "NCAS-CMS_pyfive/tests/data/issue23_A_contiguous.nc",
+  "NCAS-CMS_pyfive/tests/data/issue23_B.nc",
+  "NCAS-CMS_pyfive/tests/data/latest.hdf5",
+  "NCAS-CMS_pyfive/tests/data/netcdf4_classic.nc",
+  "NCAS-CMS_pyfive/tests/data/new_style_groups.hdf5",
+  "NCAS-CMS_pyfive/tests/data/noy_AERmonZ_UKESM1-0-LL_piControl_r1i1p1f2_gnz_200001-200012.nc",
+  "NCAS-CMS_pyfive/tests/data/references.hdf5",
+  "NCAS-CMS_pyfive/tests/data/resizable.hdf5",
+  "NCAS-CMS_pyfive/tests/opaque_datetime.hdf5",
+  "NCAS-CMS_pyfive/tests/opaque_fixed.hdf5",
+  "cve_hdf5/cvefiles/cve-2016-4330.h5",
+  "cve_hdf5/cvefiles/cve-2016-4331.h5",
+  "cve_hdf5/cvefiles/cve-2016-4332-mtime-new.h5",
+  "cve_hdf5/cvefiles/cve-2016-4332-mtime.h5",
+  "cve_hdf5/cvefiles/cve-2016-4333.h5",
+  "cve_hdf5/cvefiles/cve-2017-17505.h5",
+  "cve_hdf5/cvefiles/cve-2017-17506.h5",
+  "cve_hdf5/cvefiles/cve-2017-17507.h5",
+  "cve_hdf5/cvefiles/cve-2017-17508.h5",
+  "cve_hdf5/cvefiles/cve-2017-17509.h5",
+  "cve_hdf5/cvefiles/cve-2018-11202.h5",
+  "cve_hdf5/cvefiles/cve-2018-11203.h5",
+  "cve_hdf5/cvefiles/cve-2018-11204.h5",
+  "cve_hdf5/cvefiles/cve-2018-11205.h5",
+  "cve_hdf5/cvefiles/cve-2018-11206-new.h5",
+  "cve_hdf5/cvefiles/cve-2018-11206-old.h5",
+  "cve_hdf5/cvefiles/cve-2018-11207.h5",
+  "cve_hdf5/cvefiles/cve-2018-13867.h5",
+  "cve_hdf5/cvefiles/cve-2018-13868.h5",
+  "cve_hdf5/cvefiles/cve-2018-13869.h5",
+  "cve_hdf5/cvefiles/cve-2018-13870.h5",
+  "cve_hdf5/cvefiles/cve-2018-13871.h5",
+  "cve_hdf5/cvefiles/cve-2018-13872.h5",
+  "cve_hdf5/cvefiles/cve-2018-13873.h5",
+  "cve_hdf5/cvefiles/cve-2018-13875.h5",
+  "cve_hdf5/cvefiles/cve-2018-14031.h5",
+  "cve_hdf5/cvefiles/cve-2018-14033.h5",
+  "cve_hdf5/cvefiles/cve-2018-14034.h5",
+  "cve_hdf5/cvefiles/cve-2018-14035.h5",
+  "cve_hdf5/cvefiles/cve-2018-14460.h5",
+  "cve_hdf5/cvefiles/cve-2018-15671.h5",
+  "cve_hdf5/cvefiles/cve-2018-15672.h5",
+  "cve_hdf5/cvefiles/cve-2018-16438.h5",
+  "cve_hdf5/cvefiles/cve-2018-17233.h5",
+  "cve_hdf5/cvefiles/cve-2018-17234.h5",
+  "cve_hdf5/cvefiles/cve-2018-17237.h5",
+  "cve_hdf5/cvefiles/cve-2018-17432.h5",
+  "cve_hdf5/cvefiles/cve-2018-17434.h5",
+  "cve_hdf5/cvefiles/cve-2018-17435.h5",
+  "cve_hdf5/cvefiles/cve-2018-17437.h5",
+  "cve_hdf5/cvefiles/cve-2019-8396.h5",
+  "cve_hdf5/cvefiles/cve-2019-9152.h5",
+  "cve_hdf5/cvefiles/cve-2020-10811.h5",
+  "cve_hdf5/cvefiles/cve-2020-18232.h5",
+  "cve_hdf5/cvefiles/cve-2021-36977.h5",
+  "cve_hdf5/cvefiles/cve-2021-37501.h5",
+  "cve_hdf5/cvefiles/cve-2021-45829.h5",
+  "cve_hdf5/cvefiles/cve-2021-45833.h5",
+  "cve_hdf5/cvefiles/cve-2024-29157.h5",
+  "cve_hdf5/cvefiles/cve-2024-29158.h5",
+  "cve_hdf5/cvefiles/cve-2024-29159.h5",
+  "cve_hdf5/cvefiles/cve-2024-29160.h5",
+  "cve_hdf5/cvefiles/cve-2024-29161.h5",
+  "cve_hdf5/cvefiles/cve-2024-29162.h5",
+  "cve_hdf5/cvefiles/cve-2024-29163.h5",
+  "cve_hdf5/cvefiles/cve-2024-29164.h5",
+  "cve_hdf5/cvefiles/cve-2024-29165.h5",
+  "cve_hdf5/cvefiles/cve-2024-29166.h5",
+  "cve_hdf5/cvefiles/cve-2024-32605.h5",
+  "cve_hdf5/cvefiles/cve-2024-32606.h5",
+  "cve_hdf5/cvefiles/cve-2024-32607-1.h5",
+  "cve_hdf5/cvefiles/cve-2024-32607-2.h5",
+  "cve_hdf5/cvefiles/cve-2024-32608.h5",
+  "cve_hdf5/cvefiles/cve-2024-32610.h5",
+  "cve_hdf5/cvefiles/cve-2024-32611.h5",
+  "cve_hdf5/cvefiles/cve-2024-32612.h5",
+  "cve_hdf5/cvefiles/cve-2024-32613.h5",
+  "cve_hdf5/cvefiles/cve-2024-32614.h5",
+  "cve_hdf5/cvefiles/cve-2024-32615.h5",
+  "cve_hdf5/cvefiles/cve-2024-32616.h5",
+  "cve_hdf5/cvefiles/cve-2024-32617.h5",
+  "cve_hdf5/cvefiles/cve-2024-32619.h5",
+  "cve_hdf5/cvefiles/cve-2024-32620.h5",
+  "cve_hdf5/cvefiles/cve-2024-32621.h5",
+  "cve_hdf5/cvefiles/cve-2024-32622.h5",
+  "cve_hdf5/cvefiles/cve-2024-32624.h5",
+  "cve_hdf5/cvefiles/cve-2024-33873.h5",
+  "cve_hdf5/cvefiles/cve-2024-33875.h5",
+  "cve_hdf5/cvefiles/cve-2024-33876.h5",
+  "cve_hdf5/cvefiles/cve-2024-33877.h5",
+  "cve_hdf5/cvefiles/cve-2025-2310.h5",
+  "cve_hdf5/cvefiles/cve-2025-2924.h5",
+  "cve_hdf5/cvefiles/cve-2025-2925.h5",
+  "cve_hdf5/cvefiles/cve-2025-6269-1.h5",
+  "cve_hdf5/cvefiles/cve-2025-6269-2.h5",
+  "cve_hdf5/cvefiles/cve-2025-6269-3.h5",
+  "cve_hdf5/cvefiles/cve-2025-6269-4.h5",
+  "cve_hdf5/cvefiles/cve-2025-6516.h5",
+  "cve_hdf5/cvefiles/cve-2025-6857.h5",
+  "cve_hdf5/cvefiles/cve-2025-7067.h5",
+  "cve_hdf5/cvefiles/cve-2026-26200.h5",
+  "cve_hdf5/cvefiles/cve-2026-34734.h5",
+  "cve_hdf5/cvefiles/cve-2026-92627.h5",
+  "cve_hdf5/cvefiles/unknown-1.h5",
+  "cve_hdf5/fuzzerfiles/gh-4431-poc-03.h5",
+  "cve_hdf5/fuzzerfiles/gh-4432-poc-05.h5",
+  "cve_hdf5/fuzzerfiles/gh-4433-poc-08.h5",
+  "cve_hdf5/fuzzerfiles/gh-4435-poc-10.h5",
+  "cve_hdf5/fuzzerfiles/gh_2649_flawed.h5",
+  "cve_hdf5/fuzzerfiles/gh_2649_plain_model.h5",
+  "h5py_data/compound-dtype-complex.h5",
+  "h5py_data/vlen_string_dset.h5",
+  "h5py_data/vlen_string_dset_utc.h5",
+  "h5py_data/vlen_string_s390x.h5",
+  "hdf5/HDF5Examples/C/H5FLT/tfiles/h5ex_d_bitgroom.h5",
+  "hdf5/HDF5Examples/C/H5FLT/tfiles/h5ex_d_blosc.h5",
+  "hdf5/HDF5Examples/C/H5FLT/tfiles/h5ex_d_bshuf.h5",
+  "hdf5/HDF5Examples/C/H5FLT/tfiles/h5ex_d_bzip2.h5",
+  "hdf5/HDF5Examples/C/H5FLT/tfiles/h5ex_d_granularbr.h5",
+  "hdf5/HDF5Examples/C/H5FLT/tfiles/h5ex_d_jpeg.h5",
+  "hdf5/HDF5Examples/C/H5FLT/tfiles/h5ex_d_lz4.h5",
+  "hdf5/HDF5Examples/C/H5FLT/tfiles/h5ex_d_lzf.h5",
+  "hdf5/HDF5Examples/C/H5FLT/tfiles/h5ex_d_zstd.h5",
+  "hdf5/HDF5Examples/C/H5G/16/h5ex_g_iterate.h5",
+  "hdf5/HDF5Examples/C/H5G/16/h5ex_g_traverse.h5",
+  "hdf5/HDF5Examples/C/H5G/h5ex_g_iterate.h5",
+  "hdf5/HDF5Examples/C/H5G/h5ex_g_traverse.h5",
+  "hdf5/HDF5Examples/C/H5G/h5ex_g_visit.h5",
+  "hdf5/HDF5Examples/FORTRAN/H5G/h5ex_g_iterate.h5",
+  "hdf5/HDF5Examples/FORTRAN/H5G/h5ex_g_traverse.h5",
+  "hdf5/HDF5Examples/FORTRAN/H5G/h5ex_g_visit.h5",
+  "hdf5/HDF5Examples/JAVA/H5G/h5ex_g_iterate.h5",
+  "hdf5/HDF5Examples/JAVA/H5G/h5ex_g_visit.h5",
+  "hdf5/HDF5Examples/JAVA/compat/H5G/110/h5ex_g_iterate.h5",
+  "hdf5/HDF5Examples/JAVA/compat/H5G/110/h5ex_g_visit.h5",
+  "hdf5/HDF5Examples/JAVA/compat/H5G/h5ex_g_iterate.h5",
+  "hdf5/HDF5Examples/JAVA/compat/H5G/h5ex_g_visit.h5",
+  "hdf5/c++/test/th5s.h5",
+  "hdf5/hl/test/testfiles/test_ds_be.h5",
+  "hdf5/hl/test/testfiles/test_ds_be_new_ref-32bit.h5",
+  "hdf5/hl/test/testfiles/test_ds_be_new_ref.h5",
+  "hdf5/hl/test/testfiles/test_ds_le.h5",
+  "hdf5/hl/test/testfiles/test_ds_le_new_ref.h5",
+  "hdf5/hl/test/testfiles/test_ld.h5",
+  "hdf5/hl/test/testfiles/test_table_be.h5",
+  "hdf5/hl/test/testfiles/test_table_cray.h5",
+  "hdf5/hl/test/testfiles/test_table_le.h5",
+  "hdf5/test/testfiles/aggr.h5",
+  "hdf5/test/testfiles/bad_chunk_ndims.h5",
+  "hdf5/test/testfiles/bad_compound.h5",
+  "hdf5/test/testfiles/bad_offset.h5",
+  "hdf5/test/testfiles/be_data.h5",
+  "hdf5/test/testfiles/be_extlink1.h5",
+  "hdf5/test/testfiles/be_extlink2.h5",
+  "hdf5/test/testfiles/btree_idx_1_6.h5",
+  "hdf5/test/testfiles/btree_idx_1_8.h5",
+  "hdf5/test/testfiles/charsets.h5",
+  "hdf5/test/testfiles/corrupt_stab_msg.h5",
+  "hdf5/test/testfiles/deflate.h5",
+  "hdf5/test/testfiles/file_image_core_test.h5",
+  "hdf5/test/testfiles/filespace_1_6.h5",
+  "hdf5/test/testfiles/filespace_1_8.h5",
+  "hdf5/test/testfiles/fill18.h5",
+  "hdf5/test/testfiles/fill_old.h5",
+  "hdf5/test/testfiles/filter_error.h5",
+  "hdf5/test/testfiles/fsm_aggr_nopersist.h5",
+  "hdf5/test/testfiles/fsm_aggr_persist.h5",
+  "hdf5/test/testfiles/group_old.h5",
+  "hdf5/test/testfiles/h5fc_ext1_f.h5",
+  "hdf5/test/testfiles/h5fc_ext1_i.h5",
+  "hdf5/test/testfiles/h5fc_ext2_if.h5",
+  "hdf5/test/testfiles/h5fc_ext2_sf.h5",
+  "hdf5/test/testfiles/h5fc_ext3_isf.h5",
+  "hdf5/test/testfiles/h5fc_ext_none.h5",
+  "hdf5/test/testfiles/le_data.h5",
+  "hdf5/test/testfiles/le_extlink1.h5",
+  "hdf5/test/testfiles/le_extlink2.h5",
+  "hdf5/test/testfiles/memleak_H5O_dtype_decode_helper_H5Odtype.h5",
+  "hdf5/test/testfiles/mergemsg.h5",
+  "hdf5/test/testfiles/noencoder.h5",
+  "hdf5/test/testfiles/none.h5",
+  "hdf5/test/testfiles/paged_nopersist.h5",
+  "hdf5/test/testfiles/paged_persist.h5",
+  "hdf5/test/testfiles/specmetaread.h5",
+  "hdf5/test/testfiles/tarrold.h5",
+  "hdf5/test/testfiles/tbad_msg_count.h5",
+  "hdf5/test/testfiles/tbogus.h5",
+  "hdf5/test/testfiles/test_filters_be.h5",
+  "hdf5/test/testfiles/test_filters_le.h5",
+  "hdf5/test/testfiles/th5s.h5",
+  "hdf5/test/testfiles/tlayouto.h5",
+  "hdf5/test/testfiles/tmisc38a.h5",
+  "hdf5/test/testfiles/tmisc38b.h5",
+  "hdf5/test/testfiles/tmtimen.h5",
+  "hdf5/test/testfiles/tmtimeo.h5",
+  "hdf5/test/testfiles/tnullspace.h5",
+  "hdf5/test/testfiles/tsizeslheap.h5",
+  "hdf5/tools/test/testfiles/bigendian/tdset2.h5",
+  "hdf5/tools/test/testfiles/binfp64.h5",
+  "hdf5/tools/test/testfiles/binin16.h5",
+  "hdf5/tools/test/testfiles/binin32.h5",
+  "hdf5/tools/test/testfiles/binin8.h5",
+  "hdf5/tools/test/testfiles/binin8w.h5",
+  "hdf5/tools/test/testfiles/binuin16.h5",
+  "hdf5/tools/test/testfiles/binuin32.h5",
+  "hdf5/tools/test/testfiles/bounds_latest_latest.h5",
+  "hdf5/tools/test/testfiles/charsets.h5",
+  "hdf5/tools/test/testfiles/compounds_array_vlen1.h5",
+  "hdf5/tools/test/testfiles/compounds_array_vlen2.h5",
+  "hdf5/tools/test/testfiles/err_attr_dspace.h5",
+  "hdf5/tools/test/testfiles/file_space.h5",
+  "hdf5/tools/test/testfiles/filter_fail.h5",
+  "hdf5/tools/test/testfiles/h5clear_fsm_persist_equal.h5",
+  "hdf5/tools/test/testfiles/h5clear_fsm_persist_less.h5",
+  "hdf5/tools/test/testfiles/h5clear_fsm_persist_noclose.h5",
+  "hdf5/tools/test/testfiles/h5clear_fsm_persist_user_equal.h5",
+  "hdf5/tools/test/testfiles/h5clear_fsm_persist_user_less.h5",
+  "hdf5/tools/test/testfiles/h5clear_sec2_v0.h5",
+  "hdf5/tools/test/testfiles/h5clear_sec2_v2.h5",
+  "hdf5/tools/test/testfiles/h5copy_extlinks_src.h5",
+  "hdf5/tools/test/testfiles/h5copy_extlinks_trg.h5",
+  "hdf5/tools/test/testfiles/h5copy_ref.h5",
+  "hdf5/tools/test/testfiles/h5copytst.h5",
+  "hdf5/tools/test/testfiles/h5copytst_new.h5",
+  "hdf5/tools/test/testfiles/h5diff_attr1.h5",
+  "hdf5/tools/test/testfiles/h5diff_attr2.h5",
+  "hdf5/tools/test/testfiles/h5diff_attr3.h5",
+  "hdf5/tools/test/testfiles/h5diff_attr_v_level1.h5",
+  "hdf5/tools/test/testfiles/h5diff_attr_v_level2.h5",
+  "hdf5/tools/test/testfiles/h5diff_basic1.h5",
+  "hdf5/tools/test/testfiles/h5diff_basic2.h5",
+  "hdf5/tools/test/testfiles/h5diff_comp_vl_strs.h5",
+  "hdf5/tools/test/testfiles/h5diff_danglelinks1.h5",
+  "hdf5/tools/test/testfiles/h5diff_danglelinks2.h5",
+  "hdf5/tools/test/testfiles/h5diff_dset1.h5",
+  "hdf5/tools/test/testfiles/h5diff_dset2.h5",
+  "hdf5/tools/test/testfiles/h5diff_dset3.h5",
+  "hdf5/tools/test/testfiles/h5diff_dset_zero_dim_size1.h5",
+  "hdf5/tools/test/testfiles/h5diff_dset_zero_dim_size2.h5",
+  "hdf5/tools/test/testfiles/h5diff_dtypes.h5",
+  "hdf5/tools/test/testfiles/h5diff_empty.h5",
+  "hdf5/tools/test/testfiles/h5diff_enum_invalid_values.h5",
+  "hdf5/tools/test/testfiles/h5diff_eps1.h5",
+  "hdf5/tools/test/testfiles/h5diff_eps2.h5",
+  "hdf5/tools/test/testfiles/h5diff_exclude1-1.h5",
+  "hdf5/tools/test/testfiles/h5diff_exclude1-2.h5",
+  "hdf5/tools/test/testfiles/h5diff_exclude2-1.h5",
+  "hdf5/tools/test/testfiles/h5diff_exclude2-2.h5",
+  "hdf5/tools/test/testfiles/h5diff_exclude3-1.h5",
+  "hdf5/tools/test/testfiles/h5diff_exclude3-2.h5",
+  "hdf5/tools/test/testfiles/h5diff_ext2softlink_src.h5",
+  "hdf5/tools/test/testfiles/h5diff_ext2softlink_trg.h5",
+  "hdf5/tools/test/testfiles/h5diff_extlink_src.h5",
+  "hdf5/tools/test/testfiles/h5diff_extlink_trg.h5",
+  "hdf5/tools/test/testfiles/h5diff_grp_recurse1.h5",
+  "hdf5/tools/test/testfiles/h5diff_grp_recurse2.h5",
+  "hdf5/tools/test/testfiles/h5diff_grp_recurse_ext1.h5",
+  "hdf5/tools/test/testfiles/h5diff_grp_recurse_ext2-1.h5",
+  "hdf5/tools/test/testfiles/h5diff_grp_recurse_ext2-2.h5",
+  "hdf5/tools/test/testfiles/h5diff_grp_recurse_ext2-3.h5",
+  "hdf5/tools/test/testfiles/h5diff_hyper1.h5",
+  "hdf5/tools/test/testfiles/h5diff_hyper2.h5",
+  "hdf5/tools/test/testfiles/h5diff_linked_softlink.h5",
+  "hdf5/tools/test/testfiles/h5diff_links.h5",
+  "hdf5/tools/test/testfiles/h5diff_onion_dset_1d.h5",
+  "hdf5/tools/test/testfiles/h5diff_onion_dset_ext.h5",
+  "hdf5/tools/test/testfiles/h5diff_onion_objs.h5",
+  "hdf5/tools/test/testfiles/h5diff_softlinks.h5",
+  "hdf5/tools/test/testfiles/h5diff_strings1.h5",
+  "hdf5/tools/test/testfiles/h5diff_strings2.h5",
+  "hdf5/tools/test/testfiles/h5fc_edge_v3.h5",
+  "hdf5/tools/test/testfiles/h5fc_err_level.h5",
+  "hdf5/tools/test/testfiles/h5fc_ext1_f.h5",
+  "hdf5/tools/test/testfiles/h5fc_ext1_i.h5",
+  "hdf5/tools/test/testfiles/h5fc_ext1_s.h5",
+  "hdf5/tools/test/testfiles/h5fc_ext2_if.h5",
+  "hdf5/tools/test/testfiles/h5fc_ext2_is.h5",
+  "hdf5/tools/test/testfiles/h5fc_ext2_sf.h5",
+  "hdf5/tools/test/testfiles/h5fc_ext3_isf.h5",
+  "hdf5/tools/test/testfiles/h5fc_ext_none.h5",
+  "hdf5/tools/test/testfiles/h5fc_non_v3.h5",
+  "hdf5/tools/test/testfiles/h5repack_CVE-2018-14460.h5",
+  "hdf5/tools/test/testfiles/h5repack_CVE-2018-17432.h5",
+  "hdf5/tools/test/testfiles/h5repack_aggr.h5",
+  "hdf5/tools/test/testfiles/h5repack_attr.h5",
+  "hdf5/tools/test/testfiles/h5repack_attr_refs.h5",
+  "hdf5/tools/test/testfiles/h5repack_deflate.h5",
+  "hdf5/tools/test/testfiles/h5repack_early.h5",
+  "hdf5/tools/test/testfiles/h5repack_ext.h5",
+  "hdf5/tools/test/testfiles/h5repack_f32le.h5",
+  "hdf5/tools/test/testfiles/h5repack_f32le_ex.h5",
+  "hdf5/tools/test/testfiles/h5repack_fill.h5",
+  "hdf5/tools/test/testfiles/h5repack_filters.h5",
+  "hdf5/tools/test/testfiles/h5repack_fletcher.h5",
+  "hdf5/tools/test/testfiles/h5repack_fsm_aggr_nopersist.h5",
+  "hdf5/tools/test/testfiles/h5repack_fsm_aggr_persist.h5",
+  "hdf5/tools/test/testfiles/h5repack_hlink.h5",
+  "hdf5/tools/test/testfiles/h5repack_int32le_1d.h5",
+  "hdf5/tools/test/testfiles/h5repack_int32le_1d_ex.h5",
+  "hdf5/tools/test/testfiles/h5repack_int32le_2d.h5",
+  "hdf5/tools/test/testfiles/h5repack_int32le_2d_ex.h5",
+  "hdf5/tools/test/testfiles/h5repack_int32le_3d.h5",
+  "hdf5/tools/test/testfiles/h5repack_int32le_3d_ex.h5",
+  "hdf5/tools/test/testfiles/h5repack_layout.UD.h5",
+  "hdf5/tools/test/testfiles/h5repack_layout.h5",
+  "hdf5/tools/test/testfiles/h5repack_layout2.h5",
+  "hdf5/tools/test/testfiles/h5repack_layout3.h5",
+  "hdf5/tools/test/testfiles/h5repack_layouto.h5",
+  "hdf5/tools/test/testfiles/h5repack_named_dtypes.h5",
+  "hdf5/tools/test/testfiles/h5repack_nbit.h5",
+  "hdf5/tools/test/testfiles/h5repack_nested_8bit_enum.h5",
+  "hdf5/tools/test/testfiles/h5repack_nested_8bit_enum_deflated.h5",
+  "hdf5/tools/test/testfiles/h5repack_none.h5",
+  "hdf5/tools/test/testfiles/h5repack_objs.h5",
+  "hdf5/tools/test/testfiles/h5repack_paged_nopersist.h5",
+  "hdf5/tools/test/testfiles/h5repack_paged_persist.h5",
+  "hdf5/tools/test/testfiles/h5repack_refs.h5",
+  "hdf5/tools/test/testfiles/h5repack_shuffle.h5",
+  "hdf5/tools/test/testfiles/h5repack_soffset.h5",
+  "hdf5/tools/test/testfiles/h5repack_szip.h5",
+  "hdf5/tools/test/testfiles/h5repack_uint8be.h5",
+  "hdf5/tools/test/testfiles/h5repack_uint8be_ex.h5",
+  "hdf5/tools/test/testfiles/h5stat_err_old_fill.h5",
+  "hdf5/tools/test/testfiles/h5stat_err_old_layout.h5",
+  "hdf5/tools/test/testfiles/h5stat_err_refcount.h5",
+  "hdf5/tools/test/testfiles/h5stat_filters.h5",
+  "hdf5/tools/test/testfiles/h5stat_idx.h5",
+  "hdf5/tools/test/testfiles/h5stat_newgrat.h5",
+  "hdf5/tools/test/testfiles/h5stat_threshold.h5",
+  "hdf5/tools/test/testfiles/h5stat_tsohm.h5",
+  "hdf5/tools/test/testfiles/mod_h5clear_mdc_image.h5",
+  "hdf5/tools/test/testfiles/non_comparables1.h5",
+  "hdf5/tools/test/testfiles/non_comparables2.h5",
+  "hdf5/tools/test/testfiles/old_h5fc_ext1_f.h5",
+  "hdf5/tools/test/testfiles/old_h5fc_ext1_i.h5",
+  "hdf5/tools/test/testfiles/old_h5fc_ext1_s.h5",
+  "hdf5/tools/test/testfiles/old_h5fc_ext2_if.h5",
+  "hdf5/tools/test/testfiles/old_h5fc_ext2_is.h5",
+  "hdf5/tools/test/testfiles/old_h5fc_ext2_sf.h5",
+  "hdf5/tools/test/testfiles/old_h5fc_ext3_isf.h5",
+  "hdf5/tools/test/testfiles/old_h5fc_ext_none.h5",
+  "hdf5/tools/test/testfiles/packedbits.h5",
+  "hdf5/tools/test/testfiles/t128bit_float.h5",
+  "hdf5/tools/test/testfiles/tCVE-2021-37501_attr_decode.h5",
+  "hdf5/tools/test/testfiles/tCVE_2018_11206_fill_new.h5",
+  "hdf5/tools/test/testfiles/tCVE_2018_11206_fill_old.h5",
+  "hdf5/tools/test/testfiles/taindices.h5",
+  "hdf5/tools/test/testfiles/tarray1.h5",
+  "hdf5/tools/test/testfiles/tarray1_big.h5",
+  "hdf5/tools/test/testfiles/tarray2.h5",
+  "hdf5/tools/test/testfiles/tarray4.h5",
+  "hdf5/tools/test/testfiles/tarray5.h5",
+  "hdf5/tools/test/testfiles/tarray8.h5",
+  "hdf5/tools/test/testfiles/tattr.h5",
+  "hdf5/tools/test/testfiles/tattr2.h5",
+  "hdf5/tools/test/testfiles/tattr4_be.h5",
+  "hdf5/tools/test/testfiles/tattrintsize.h5",
+  "hdf5/tools/test/testfiles/tattrreg.h5",
+  "hdf5/tools/test/testfiles/tbfloat16.h5",
+  "hdf5/tools/test/testfiles/tbfloat16_be.h5",
+  "hdf5/tools/test/testfiles/tbigdims.h5",
+  "hdf5/tools/test/testfiles/tbinary.h5",
+  "hdf5/tools/test/testfiles/tbitnopaque.h5",
+  "hdf5/tools/test/testfiles/tchar.h5",
+  "hdf5/tools/test/testfiles/tcmpdattrintsize.h5",
+  "hdf5/tools/test/testfiles/tcmpdintarray.h5",
+  "hdf5/tools/test/testfiles/tcmpdints.h5",
+  "hdf5/tools/test/testfiles/tcmpdintsize.h5",
+  "hdf5/tools/test/testfiles/tcomplex.h5",
+  "hdf5/tools/test/testfiles/tcompound.h5",
+  "hdf5/tools/test/testfiles/tcompound_complex.h5",
+  "hdf5/tools/test/testfiles/tcompound_complex2.h5",
+  "hdf5/tools/test/testfiles/tdatareg.h5",
+  "hdf5/tools/test/testfiles/tdset.h5",
+  "hdf5/tools/test/testfiles/tdset2.h5",
+  "hdf5/tools/test/testfiles/tdset_idx.h5",
+  "hdf5/tools/test/testfiles/tempty.h5",
+  "hdf5/tools/test/testfiles/textlink.h5",
+  "hdf5/tools/test/testfiles/textlinkfar.h5",
+  "hdf5/tools/test/testfiles/textlinksrc.h5",
+  "hdf5/tools/test/testfiles/textlinktar.h5",
+  "hdf5/tools/test/testfiles/textpfe.h5",
+  "hdf5/tools/test/testfiles/tfcontents2.h5",
+  "hdf5/tools/test/testfiles/tfilters.h5",
+  "hdf5/tools/test/testfiles/tfloat16.h5",
+  "hdf5/tools/test/testfiles/tfloat16_be.h5",
+  "hdf5/tools/test/testfiles/tfloat4.h5",
+  "hdf5/tools/test/testfiles/tfloat6.h5",
+  "hdf5/tools/test/testfiles/tfloat8.h5",
+  "hdf5/tools/test/testfiles/tfloatsattrs.h5",
+  "hdf5/tools/test/testfiles/tfpformat.h5",
+  "hdf5/tools/test/testfiles/tfvalues.h5",
+  "hdf5/tools/test/testfiles/tgroup.h5",
+  "hdf5/tools/test/testfiles/tgrp_comments.h5",
+  "hdf5/tools/test/testfiles/tgrpnullspace.h5",
+  "hdf5/tools/test/testfiles/thlink.h5",
+  "hdf5/tools/test/testfiles/thyperslab.h5",
+  "hdf5/tools/test/testfiles/tintascii.h5",
+  "hdf5/tools/test/testfiles/tints4dims.h5",
+  "hdf5/tools/test/testfiles/tintsattrs.h5",
+  "hdf5/tools/test/testfiles/tintsnodata.h5",
+  "hdf5/tools/test/testfiles/tlarge_objname.h5",
+  "hdf5/tools/test/testfiles/tldouble.h5",
+  "hdf5/tools/test/testfiles/tldouble_scalar.h5",
+  "hdf5/tools/test/testfiles/tlonglinks.h5",
+  "hdf5/tools/test/testfiles/tloop.h5",
+  "hdf5/tools/test/testfiles/tnamed_dtype_attr.h5",
+  "hdf5/tools/test/testfiles/tnestedcmpddt.h5",
+  "hdf5/tools/test/testfiles/tnestedcomp.h5",
+  "hdf5/tools/test/testfiles/tno-subset.h5",
+  "hdf5/tools/test/testfiles/tnullspace.h5",
+  "hdf5/tools/test/testfiles/torderattr.h5",
+  "hdf5/tools/test/testfiles/tordergr.h5",
+  "hdf5/tools/test/testfiles/trefer_attr.h5",
+  "hdf5/tools/test/testfiles/trefer_compat.h5",
+  "hdf5/tools/test/testfiles/trefer_ext1.h5",
+  "hdf5/tools/test/testfiles/trefer_ext2.h5",
+  "hdf5/tools/test/testfiles/trefer_grp.h5",
+  "hdf5/tools/test/testfiles/trefer_obj.h5",
+  "hdf5/tools/test/testfiles/trefer_obj_del.h5",
+  "hdf5/tools/test/testfiles/trefer_param.h5",
+  "hdf5/tools/test/testfiles/trefer_reg.h5",
+  "hdf5/tools/test/testfiles/trefer_reg_1d.h5",
+  "hdf5/tools/test/testfiles/tsaf.h5",
+  "hdf5/tools/test/testfiles/tscalarattrintsize.h5",
+  "hdf5/tools/test/testfiles/tscalarintattrsize.h5",
+  "hdf5/tools/test/testfiles/tscalarintsize.h5",
+  "hdf5/tools/test/testfiles/tscalarstring.h5",
+  "hdf5/tools/test/testfiles/tslink.h5",
+  "hdf5/tools/test/testfiles/tsoftlinks.h5",
+  "hdf5/tools/test/testfiles/tst_onion_dset_1d.h5",
+  "hdf5/tools/test/testfiles/tst_onion_dset_ext.h5",
+  "hdf5/tools/test/testfiles/tst_onion_objs.h5",
+  "hdf5/tools/test/testfiles/tstr.h5",
+  "hdf5/tools/test/testfiles/tstr2.h5",
+  "hdf5/tools/test/testfiles/tstr3.h5",
+  "hdf5/tools/test/testfiles/tudfilter.h5",
+  "hdf5/tools/test/testfiles/tudfilter2.h5",
+  "hdf5/tools/test/testfiles/tvldtypes1.h5",
+  "hdf5/tools/test/testfiles/tvldtypes2.h5",
+  "hdf5/tools/test/testfiles/tvldtypes3.h5",
+  "hdf5/tools/test/testfiles/tvldtypes4.h5",
+  "hdf5/tools/test/testfiles/tvldtypes5.h5",
+  "hdf5/tools/test/testfiles/tvlenstr_array.h5",
+  "hdf5/tools/test/testfiles/tvlstr.h5",
+  "hdf5/tools/test/testfiles/tvms.h5",
+  "hdf5/tools/test/testfiles/txtfp32.h5",
+  "hdf5/tools/test/testfiles/txtfp64.h5",
+  "hdf5/tools/test/testfiles/txtin16.h5",
+  "hdf5/tools/test/testfiles/txtin32.h5",
+  "hdf5/tools/test/testfiles/txtin8.h5",
+  "hdf5/tools/test/testfiles/txtstr.h5",
+  "hdf5/tools/test/testfiles/txtuin16.h5",
+  "hdf5/tools/test/testfiles/txtuin32.h5",
+  "hdf5/tools/test/testfiles/vds/1_a.h5",
+  "hdf5/tools/test/testfiles/vds/1_b.h5",
+  "hdf5/tools/test/testfiles/vds/1_c.h5",
+  "hdf5/tools/test/testfiles/vds/1_d.h5",
+  "hdf5/tools/test/testfiles/vds/1_e.h5",
+  "hdf5/tools/test/testfiles/vds/1_f.h5",
+  "hdf5/tools/test/testfiles/vds/1_vds.h5",
+  "hdf5/tools/test/testfiles/vds/2_a.h5",
+  "hdf5/tools/test/testfiles/vds/2_b.h5",
+  "hdf5/tools/test/testfiles/vds/2_c.h5",
+  "hdf5/tools/test/testfiles/vds/2_d.h5",
+  "hdf5/tools/test/testfiles/vds/2_e.h5",
+  "hdf5/tools/test/testfiles/vds/2_vds.h5",
+  "hdf5/tools/test/testfiles/vds/3_1_vds.h5",
+  "hdf5/tools/test/testfiles/vds/3_2_vds.h5",
+  "hdf5/tools/test/testfiles/vds/4_0.h5",
+  "hdf5/tools/test/testfiles/vds/4_1.h5",
+  "hdf5/tools/test/testfiles/vds/4_2.h5",
+  "hdf5/tools/test/testfiles/vds/4_vds.h5",
+  "hdf5/tools/test/testfiles/vds/5_a.h5",
+  "hdf5/tools/test/testfiles/vds/5_b.h5",
+  "hdf5/tools/test/testfiles/vds/5_c.h5",
+  "hdf5/tools/test/testfiles/vds/5_vds.h5",
+  "hdf5/tools/test/testfiles/vds/a.h5",
+  "hdf5/tools/test/testfiles/vds/b.h5",
+  "hdf5/tools/test/testfiles/vds/c.h5",
+  "hdf5/tools/test/testfiles/vds/d.h5",
+  "hdf5/tools/test/testfiles/vds/f-0.h5",
+  "hdf5/tools/test/testfiles/vds/f-3.h5",
+  "hdf5/tools/test/testfiles/vds/vds-eiger.h5",
+  "hdf5/tools/test/testfiles/vds/vds-percival-unlim-maxmin.h5",
+  "hdf5/tools/test/testfiles/xml/tbitfields.h5",
+  "hdf5/tools/test/testfiles/xml/tcompound2.h5",
+  "hdf5/tools/test/testfiles/xml/tdset2.h5",
+  "hdf5/tools/test/testfiles/xml/tenum.h5",
+  "hdf5/tools/test/testfiles/xml/test35.nc",
+  "hdf5/tools/test/testfiles/xml/tloop2.h5",
+  "hdf5/tools/test/testfiles/xml/tname-amp.h5",
+  "hdf5/tools/test/testfiles/xml/tname-apos.h5",
+  "hdf5/tools/test/testfiles/xml/tname-gt.h5",
+  "hdf5/tools/test/testfiles/xml/tname-lt.h5",
+  "hdf5/tools/test/testfiles/xml/tname-quot.h5",
+  "hdf5/tools/test/testfiles/xml/tname-sp.h5",
+  "hdf5/tools/test/testfiles/xml/tnodata.h5",
+  "hdf5/tools/test/testfiles/xml/tobjref.h5",
+  "hdf5/tools/test/testfiles/xml/topaque.h5",
+  "hdf5/tools/test/testfiles/xml/tref-escapes-at.h5",
+  "hdf5/tools/test/testfiles/xml/tref-escapes.h5",
+  "hdf5/tools/test/testfiles/xml/tref.h5",
+  "hdf5/tools/test/testfiles/xml/tstring-at.h5",
+  "hdf5/tools/test/testfiles/xml/tstring.h5",
+  "hdf5/tools/test/testfiles/zerodim.h5",
+  "netcdf-c/h5_test/ref_tst_h_compounds.h5",
+  "netcdf-c/h5_test/ref_tst_h_compounds2.h5",
+  "netcdf-c/nc_test4/ref_hdf5_compat1.nc",
+  "netcdf-c/nc_test4/ref_hdf5_compat2.nc",
+  "netcdf-c/nc_test4/ref_hdf5_compat3.nc",
+  "netcdf-c/nc_test4/ref_szip.h5",
+  "netcdf-c/nc_test4/ref_tst_compounds.nc",
+  "netcdf-c/nc_test4/ref_tst_dims.nc",
+  "netcdf-c/nc_test4/ref_tst_interops4.nc",
+  "netcdf-c/nc_test4/ref_tst_xplatform2_1.nc",
+  "netcdf-c/nc_test4/ref_tst_xplatform2_2.nc",
+  "netcdf-c/nc_test4/tdset.h5",
+  "netcdf-c/ncdump/ref_nc_test_netcdf4_4_0.nc",
+  "netcdf-c/ncdump/ref_no_ncproperty.nc",
+  "netcdf-c/ncdump/ref_provenance_v1.nc",
+  "netcdf-c/ncdump/ref_test_corrupt_magic.nc",
+  "netcdf-c/ncdump/ref_tst_compounds2.nc",
+  "netcdf-c/ncdump/ref_tst_compounds3.nc",
+  "netcdf-c/ncdump/ref_tst_compounds4.nc",
+  "netcdf-c/ncdump/ref_tst_irish_rover.nc",
+  "netcdf4-python/examples/data/prmsl.2000.nc",
+  "netcdf4-python/examples/data/prmsl.2001.nc",
+  "netcdf4-python/examples/data/prmsl.2002.nc",
+  "netcdf4-python/examples/data/prmsl.2003.nc",
+  "netcdf4-python/examples/data/prmsl.2004.nc",
+  "netcdf4-python/examples/data/prmsl.2005.nc",
+  "netcdf4-python/examples/data/prmsl.2006.nc",
+  "netcdf4-python/examples/data/prmsl.2007.nc",
+  "netcdf4-python/examples/data/prmsl.2008.nc",
+  "netcdf4-python/examples/data/prmsl.2009.nc",
+  "netcdf4-python/examples/data/prmsl.2010.nc",
+  "netcdf4-python/examples/data/prmsl.2011.nc",
+  "netcdf4-python/examples/data/rtofs_glo_3dz_f006_6hrly_reg3.nc",
+  "netcdf4-python/test/20171025_2056.Cloud_Top_Height.nc",
+  "netcdf4-python/test/issue1152.nc",
+  "netcdf4-python/test/issue671.nc",
+  "netcdf4-python/test/issue672.nc",
+  "netcdf4-python/test/test_gold.nc",
+  "usnistgov_h5wasm/test/array.h5",
+  "usnistgov_h5wasm/test/compressed.h5",
+  "usnistgov_h5wasm/test/empty.h5",
+  "usnistgov_h5wasm/test/float16.h5",
+  "usnistgov_h5wasm/test/vlen.h5",
+  "xarray-data/ROMS_example.nc",
+  "xarray-data/basin_mask.nc",
+  "xarray-data/imerghh_730.hdf5",
+  "xarray-data/precipitation.nc4"
+ ]
+}

conformance/check.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python3
+"""check.py <results_dir> <baseline.json> [--update]
+
+The conformance gate. Fails (exit 1) when
+  * clawhdf5 panicked, hung, crashed or ran out of memory on any file, or
+  * the ok count fell below the baseline's, or
+  * a file the baseline lists as ok is no longer ok (even if another file
+    became ok and the total held).
+New ok files are reported so the baseline can be raised (--update rewrites it
+from the results).
+"""
+import json
+import os
+import sys
+
+FATAL = ("panic", "hang", "crash", "oom")
+
+
+def main():
+    args = [a for a in sys.argv[1:] if not a.startswith("--")]
+    update = "--update" in sys.argv
+    res_dir, base_path = args
+    res = json.load(open(os.path.join(res_dir, "results.json")))
+    rows = res["rows"]
+    counts = {}
+    per_corpus = {}
+    for r in rows:
+        counts[r["class"]] = counts.get(r["class"], 0) + 1
+        pc = per_corpus.setdefault(r["corpus"], {})
+        pc[r["class"]] = pc.get(r["class"], 0) + 1
+    ok_files = sorted(r["file"] for r in rows if r["class"] == "ok")
+
+    if update:
+        meta = {}
+        mp = os.path.join(res_dir, "report-meta.json")
+        if os.path.exists(mp):
+            meta = json.load(open(mp))
+        base = {
+            "comment": "conformance/run.sh fails if the ok count drops below `ok` or a file in `ok_files` stops being ok. "
+                       "Regenerate with `conformance/run.sh --update-baseline` after an intended change.",
+            "commit": meta.get("commit", ""),
+            "date": meta.get("date", ""),
+            "reference": meta.get("reference", ""),
+            "files": len(rows),
+            "ok": len(ok_files),
+            "counts": dict(sorted(counts.items())),
+            "per_corpus": {k: dict(sorted(v.items())) for k, v in sorted(per_corpus.items())},
+            "ok_files": ok_files,
+        }
+        with open(base_path, "w") as fh:
+            json.dump(base, fh, indent=1)
+            fh.write("\n")
+        print(f"baseline updated: {len(ok_files)} ok of {len(rows)} files -> {base_path}")
+        return 0
+
+    base = json.load(open(base_path))
+    failures = []
+    fatal = [r for r in rows if r["class"] in FATAL]
+    for r in fatal:
+        failures.append(f"{r['class']}: {r['file']}: {r['ours_detail'][:200]}")
+    if len(ok_files) < base["ok"]:
+        failures.append(f"ok count dropped: {len(ok_files)} < baseline {base['ok']}")
+    now_ok = set(ok_files)
+    by_file = {r["file"]: r for r in rows}
+    for f in base["ok_files"]:
+        if f not in now_ok:
+            r = by_file.get(f)
+            why = f"now {r['class']}: {(r['ours_detail'] or r['first_issue'])[:200]}" if r else "no longer in the corpus"
+            failures.append(f"regressed: {f}: {why}")
+    gained = sorted(now_ok - set(base["ok_files"]))
+
+    print(f"conformance: {len(ok_files)} ok of {len(rows)} files (baseline {base['ok']} of {base['files']}); "
+          + ", ".join(f"{k} {v}" for k, v in sorted(counts.items())))
+    if gained:
+        print(f"{len(gained)} file(s) newly ok — raise the baseline with `conformance/run.sh --update-baseline`:")
+        for f in gained:
+            print(f"  + {f}")
+    if failures:
+        print(f"CONFORMANCE GATE FAILED ({len(failures)}):")
+        for f in failures:
+            print(f"  - {f}")
+        return 1
+    print("conformance gate passed")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

conformance/compare.py

@@ -0,0 +1,289 @@
+#!/usr/bin/env python3
+"""compare.py <results_dir>: classify each file and group failures by root cause.
+
+Writes <results_dir>/results.csv, results.json and summary.md.
+File classes (first match wins):
+  hang, oom, crash, panic       ours: timeout / allocation failure / signal / any panic (caught or not)
+  h5py-cannot-read              libhdf5/h5py failed to open the file (or crashed/hung)
+  our-error                     we fail to open, list, or read something h5py reads
+  mismatch                      we read something with different shape/values, or a different object set
+  ok
+"""
+import collections
+import csv
+import json
+import os
+import re
+import sys
+
+R = sys.argv[1]
+RUNS = os.path.join(R, "runs")
+
+
+def load(d, name):
+    rc_p = os.path.join(d, name + ".rc")
+    if not os.path.exists(rc_p):
+        return None
+    rc = int(open(rc_p).read().strip() or -1)
+    err = open(os.path.join(d, name + ".err"), errors="replace").read()
+    js = None
+    try:
+        js = json.load(open(os.path.join(d, name + ".json")))
+    except Exception:  # noqa: BLE001
+        pass
+    return {"rc": rc, "err": err, "json": js}
+
+
+def proc_status(p):
+    """-> (status, detail)"""
+    if p is None:
+        return "missing", ""
+    rc, err = p["rc"], p["err"]
+    first_panic = next((ln for ln in err.splitlines() if ln.startswith("PANIC:") or "panicked at" in ln), "")
+    if rc == 0 and p["json"] is not None:
+        return "ok", ""
+    if rc == 137 or rc == 124:
+        return "hang", f"timeout ({os.environ.get('TMO', '20')} s)"
+    if "memory allocation of" in err or "MemoryError" in err or "std::bad_alloc" in err:
+        m = re.search(r"memory allocation of \d+ bytes failed", err)
+        return "oom", m.group(0) if m else "allocation failure"
+    if "overflowed its stack" in err:
+        return "crash", "stack overflow"
+    if rc == 101:
+        return "panic", first_panic or (err.strip().splitlines() or [""])[-1]
+    if rc in (134, 139, 136, 135, 132) or rc > 128:
+        sig = {134: "SIGABRT", 139: "SIGSEGV", 136: "SIGFPE", 135: "SIGBUS", 132: "SIGILL"}.get(rc, f"signal {rc - 128}")
+        tail = [ln for ln in err.strip().splitlines() if ln.strip()][-1:]
+        return "crash", f"{sig}: {tail[0][:200] if tail else ''}"
+    tail = [ln for ln in err.strip().splitlines() if ln.strip()][-1:]
+    return "crash", f"rc={rc}: {tail[0][:200] if tail else ''}"
+
+
+def norm(msg):
+    m = msg.split("\n")[0]
+    m = re.sub(r"0x[0-9a-fA-F]+", "X", m)
+    m = re.sub(r'"[^"]*"', '"…"', m)
+    m = re.sub(r"'[^']*'", "'…'", m)
+    m = re.sub(r"\d+", "N", m)
+    return m[:160]
+
+
+def panic_head(msg):
+    """First line + first clawhdf5 frame of a PANIC record."""
+    lines = msg.split("\n")
+    frame = next((ln.strip() for ln in lines[1:] if "clawhdf5_format" in ln), "")
+    return lines[0][:300], frame[:300]
+
+
+def eq_shape(a, b):
+    return a == b
+
+
+rows = []
+issues_by_file = {}
+root_causes = collections.defaultdict(lambda: {"files": set(), "count": 0, "examples": []})
+mismatch_causes = collections.defaultdict(lambda: {"files": set(), "count": 0, "examples": []})
+panics = []
+ref_only_errors = collections.Counter()
+incomparable = collections.Counter()
+
+
+def add(bucket, key, file, example):
+    b = bucket[key]
+    b["count"] += 1
+    if file not in b["files"] and len(b["examples"]) < 6:
+        b["examples"].append(example)
+    b["files"].add(file)
+
+
+files = [ln.strip() for ln in open(os.path.join(R, "files.txt")) if ln.strip()]
+for rel in files:
+    d = os.path.join(RUNS, rel.replace("/", "__"))
+    corpus = rel.split("/")[0]
+    ours, ref = load(d, "ours"), load(d, "ref")
+    h5dump = load(d, "h5dump")
+    os_, od = proc_status(ours)
+    rs, rd = proc_status(ref)
+    oj = ours["json"] if ours else None
+    rj = ref["json"] if ref else None
+    issues = []  # (kind, detail)
+    caught_panics = []
+
+    def scan_err(path, what, msg):
+        if msg.startswith("PANIC:"):
+            caught_panics.append((path, what, msg))
+
+    if oj:
+        for o in oj.get("objects", []):
+            for k in ("error", "attrs_error", "list_error"):
+                if k in o:
+                    scan_err(o["path"], k, o[k])
+            for an, av in (o.get("attrs") or {}).items():
+                if "error" in av:
+                    scan_err(o["path"], f"attr {an}", av["error"])
+        if oj.get("open_error", "").startswith("PANIC:"):
+            caught_panics.append(("<open>", "open", oj["open_error"]))
+
+    ref_open_fail = rs != "ok" or (rj is not None and "open_error" in rj)
+    ours_open_err = oj.get("open_error") if oj else None
+    n_obj = n_ok = 0
+    if os_ == "ok" and rj and not ref_open_fail and not ours_open_err:
+        ro = {x["path"]: x for x in rj.get("objects", [])}
+        oo = {x["path"]: x for x in oj.get("objects", [])}
+        our_list_errors = [x for x in oo.values() if "list_error" in x]
+        for p in sorted(set(ro) | set(oo)):
+            a, b = ro.get(p), oo.get(p)
+            n_obj += 1
+            if a is None:
+                issues.append(("mismatch", f"extra object {p} (kind={b.get('kind')})", "extra-object", b))
+                continue
+            if b is None:
+                if our_list_errors:
+                    continue  # accounted for by the list_error
+                issues.append(("mismatch", f"missing object {p} (kind={a.get('kind')})", "missing-object", a))
+                continue
+            ok = True
+            if a.get("kind") != b.get("kind") and "error" not in b and "error" not in a:
+                issues.append(("mismatch", f"{p}: kind {a.get('kind')} vs ours {b.get('kind')}", "kind", b))
+                ok = False
+            for k in ("error", "list_error", "attrs_error"):
+                if k in b and k not in a:
+                    issues.append(("our-error", f"{p}: {k}: {b[k]}", b[k], b))
+                    ok = False
+                elif k in a and k not in b and k == "error":
+                    ref_only_errors[norm(a[k])] += 1
+            if a.get("kind") == "dataset" and "error" not in a and "error" not in b:
+                if "skipped" in a or "skipped" in b:
+                    pass
+                elif a.get("converted"):
+                    incomparable[f"dataset {a['converted']}"] += 1
+                elif a.get("shape") != b.get("shape"):
+                    issues.append(("mismatch", f"{p}: shape {a.get('shape')} vs ours {b.get('shape')}", "shape", b))
+                    ok = False
+                elif a.get("hash") != b.get("hash"):
+                    issues.append(("mismatch", f"{p}: values differ (h5py {a.get('dtype')} vs ours {b.get('dtype')})", "values", b | {"ref_head": a.get("head"), "ref_dtype": a.get("dtype")}))
+                    ok = False
+            ra, oa = a.get("attrs") or {}, b.get("attrs") or {}
+            if "attrs_error" not in b and "attrs_error" not in a:
+                for an in sorted(set(ra) | set(oa)):
+                    x, y = ra.get(an), oa.get(an)
+                    if x is None:
+                        issues.append(("mismatch", f"{p}@{an}: extra attribute", "extra-attr", y or {}))
+                    elif y is None:
+                        issues.append(("mismatch", f"{p}@{an}: missing attribute", "missing-attr", x))
+                    elif "error" in y and "error" not in x:
+                        issues.append(("our-error", f"{p}@{an}: {y['error']}", y["error"], y))
+                    elif "error" in x:
+                        continue
+                    elif x.get("converted"):
+                        incomparable[f"attr {x['converted']}"] += 1
+                    elif x.get("shape") != y.get("shape"):
+                        issues.append(("mismatch", f"{p}@{an}: attr shape {x.get('shape')} vs ours {y.get('shape')}", "attr-shape", y | {"ref_dtype": x.get("dtype")}))
+                    elif x.get("hash") != y.get("hash"):
+                        issues.append(("mismatch", f"{p}@{an}: attr values differ (h5py {x.get('dtype')} vs ours {y.get('dtype')})", "attr-values", y | {"ref_head": x.get("head"), "ref_dtype": x.get("dtype")}))
+            if ok:
+                n_ok += 1
+
+    # classify
+    if os_ in ("hang", "oom", "crash", "panic"):
+        cls = os_
+    elif caught_panics:
+        cls = "panic"
+    elif ref_open_fail:
+        cls = "h5py-cannot-read"
+    elif ours_open_err:
+        cls = "our-error"
+        issues.append(("our-error", f"open: {ours_open_err}", ours_open_err, {}))
+    elif any(i[0] == "our-error" for i in issues):
+        cls = "our-error"
+    elif issues:
+        cls = "mismatch"
+    else:
+        cls = "ok"
+
+    if os_ in ("hang", "oom", "crash", "panic") or caught_panics:
+        panics.append({
+            "file": rel, "class": cls, "detail": od,
+            "stderr": (ours["err"] if ours else "")[:3000],
+            "caught": [(p, w, m[:2500]) for p, w, m in caught_panics[:3]],
+            "n_caught": len(caught_panics),
+        })
+    for kind, detail, key, rec in issues:
+        if kind == "our-error":
+            add(root_causes, norm(key), rel, detail[:300])
+        else:
+            if key in ("values", "attr-values", "shape", "attr-shape"):
+                mk = f"{key}: ours={rec.get('dtype')} h5py={rec.get('ref_dtype')} layout={rec.get('layout','-')} filters={rec.get('filters','-')}"
+            else:
+                mk = key
+            add(mismatch_causes, mk, rel, detail[:300] + (f" | ref_head={rec.get('ref_head')} our_head={rec.get('head')}" if rec.get("ref_head") else ""))
+    ref_detail = rd if rs != "ok" else ((rj or {}).get("open_error") or "")
+    h5d = ""
+    if h5dump:
+        rc = h5dump["rc"]
+        h5d = {0: "ok", 1: "error", 137: "hang", 124: "hang", 134: "SIGABRT", 139: "SIGSEGV", 136: "SIGFPE", 135: "SIGBUS"}.get(rc, f"rc={rc}")
+        if "memory allocation" in h5dump["err"] or "Cannot allocate" in h5dump["err"]:
+            h5d += "(oom)"
+    rows.append({
+        "file": rel, "corpus": corpus, "class": cls,
+        "ours": os_ if os_ != "ok" else ("open-error" if ours_open_err else ("panic" if caught_panics else "ok")),
+        "ours_detail": (od or ours_open_err or (caught_panics[0][2].split("\n")[0] if caught_panics else ""))[:300],
+        "ref": rs if rs != "ok" else ("open-error" if (rj or {}).get("open_error") else "ok"),
+        "ref_detail": ref_detail[:300],
+        "h5dump_1_14_6": h5d,
+        "h5dump_detail": ([ln for ln in h5dump["err"].splitlines() if ln.strip()][-1:] or [""])[0][:200] if h5dump else "",
+        "objects": n_obj, "objects_ok": n_ok,
+        "issues": len(issues), "first_issue": issues[0][1][:300] if issues else "",
+        "superblock": (oj or {}).get("superblock_version", ""),
+    })
+    # the first issues of each file, for report.py's known-cause matching
+    issues_by_file[rel] = [
+        {"kind": k, "key": key, "detail": det[:300], "ours_dtype": rec.get("dtype"), "ref_dtype": rec.get("ref_dtype")}
+        for k, det, key, rec in issues[:50]
+    ]
+
+with open(os.path.join(R, "results.csv"), "w", newline="") as fh:
+    w = csv.DictWriter(fh, fieldnames=list(rows[0].keys()))
+    w.writeheader()
+    w.writerows(rows)
+
+
+def ser(b):
+    return {k: {"files": len(v["files"]), "count": v["count"], "examples": v["examples"], "file_list": sorted(v["files"])} for k, v in sorted(b.items(), key=lambda kv: -len(kv[1]["files"]))}
+
+
+json.dump({"rows": rows, "issues": issues_by_file, "root_causes": ser(root_causes), "mismatch_causes": ser(mismatch_causes),
+           "panics": panics, "incomparable": incomparable.most_common(), "ref_only_errors": ref_only_errors.most_common()},
+          open(os.path.join(R, "results.json"), "w"), indent=1)
+
+classes = ["ok", "our-error", "mismatch", "h5py-cannot-read", "hang", "panic", "crash", "oom"]
+by_corpus = collections.defaultdict(collections.Counter)
+for r in rows:
+    by_corpus[r["corpus"]][r["class"]] += 1
+    by_corpus["ALL"][r["class"]] += 1
+lines = ["# Conformance sweep summary", "", "| corpus | files | " + " | ".join(classes) + " |", "|---" * (len(classes) + 2) + "|"]
+for c in sorted(by_corpus, key=lambda k: (k == "ALL", k)):
+    cnt = by_corpus[c]
+    lines.append(f"| {c} | {sum(cnt.values())} | " + " | ".join(str(cnt.get(k, 0)) for k in classes) + " |")
+lines += ["", "## Panics / hangs / crashes / OOM", ""]
+for p in panics:
+    lines.append(f"- **{p['file']}** [{p['class']}] {p['detail']}")
+    for path, what, m in p["caught"][:1]:
+        lines.append("  ```\n  " + f"{path} ({what}): " + m.replace("\n", "\n  ")[:1500] + "\n  ```")
+    if not p["caught"] and p["stderr"]:
+        lines.append("  ```\n  " + p["stderr"].strip()[:1500].replace("\n", "\n  ") + "\n  ```")
+lines += ["", "## Our-error root causes (files affected)", ""]
+for k, v in ser(root_causes).items():
+    lines.append(f"- [{v['files']} files, {v['count']} objs] `{k}`")
+    for ex in v["examples"][:3]:
+        lines.append(f"    - {ex}")
+lines += ["", "## Mismatch root causes", ""]
+for k, v in ser(mismatch_causes).items():
+    lines.append(f"- [{v['files']} files, {v['count']} objs] `{k}`")
+    for ex in v["examples"][:3]:
+        lines.append(f"    - {ex}")
+lines += ["", "## Objects h5py fails on but we read (top)", ""]
+for k, n in ref_only_errors.most_common(15):
+    lines.append(f"- {n} x `{k}`")
+open(os.path.join(R, "summary.md"), "w").write("\n".join(lines) + "\n")
+print("\n".join(lines[:4 + len(by_corpus)]))

conformance/corpus.txt

@@ -0,0 +1,19 @@
+# Conformance corpora, pinned by commit. fetch-corpus.sh reads this file.
+#
+# name  git-url  commit  root  [sparse-checkout patterns...]
+#
+# `root` is the directory inside the checkout that is swept ("." = all of it).
+# Patterns are git non-cone sparse-checkout patterns; none = whole repository.
+# Every file under <root> with an HDF5/netCDF-4 extension is probed; for
+# cve_hdf5 the extension-less files in cvefiles/ and fuzzerfiles/ are too.
+# Licences: each corpus keeps its upstream licence; nothing here is committed
+# to this repository — the files are downloaded into the gitignored cache.
+hdf5              https://github.com/HDFGroup/hdf5.git            a3cf1ea82cc7a66e50029a688121e1b105a7ce88  .  *.h5 *.he5 *.nc *.hdf5 *.h5f
+cve_hdf5          https://github.com/HDFGroup/cve_hdf5.git        3fd1f5ae3869e01b8ae02b41d7108de7ffb1a374  .
+netcdf-c          https://github.com/Unidata/netcdf-c.git         beb7b9585273c1548386231a59b809d906359033  .  /nc_test4/*.nc /ncdump/*.nc /nc_test4/*.h5 /ncdump/*.h5 /h5_test/*.h5 /hdf5_test/*.h5
+NCAS-CMS_pyfive   https://github.com/NCAS-CMS/pyfive.git          8cf07b8749133f41c5e30b8a4c604486f687fe74  .  *.h5 *.hdf5 *.hdf *.nc *.he5
+usnistgov_h5wasm  https://github.com/usnistgov/h5wasm.git         02f6336527d2812783fcedabfbf42127ec8d06d2  .  *.h5 *.hdf5 *.hdf *.nc *.he5
+netcdf4-python    https://github.com/Unidata/netcdf4-python.git   6e67576d39aef8091fb20bd767b4f1a52ddc1bec  .  *.nc *.h5
+xarray-data       https://github.com/pydata/xarray-data.git       a35297e9da2cc99c811014f0c8a4297345a5c28d  .  /basin_mask.nc /precipitation.nc4 /imerghh_730.hdf5 /eraint_uvz.nc /ROMS_example.nc /tiny.nc
+# h5py 3.16.0 (tag 3.16.0), its test data files.
+h5py_data         https://github.com/h5py/h5py.git                b2f0347c4200333acd89b43733f1caa0c115162f  h5py/tests/data_files  /h5py/tests/data_files/*

conformance/fetch-corpus.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+# fetch-corpus.sh [cache_dir]
+#
+# Download the corpora pinned in conformance/corpus.txt into the (gitignored)
+# cache: <cache>/src/<name> is a shallow, sparse, blob-filtered checkout of the
+# pinned commit and <cache>/corpus/<name> links to the swept root inside it.
+# A corpus already checked out at its pinned commit is left alone, so a second
+# run costs nothing and needs no network.
+set -euo pipefail
+HERE="$(cd "$(dirname "$0")" && pwd)"
+CACHE="${1:-${CONFORMANCE_CACHE:-$HERE/.cache}}"
+mkdir -p "$CACHE/src" "$CACHE/corpus"
+CACHE="$(cd "$CACHE" && pwd)"
+
+retry() { local i; for i in 1 2 3 4; do "$@" && return 0; sleep $((i * 5)); done; return 1; }
+
+grep -v '^[[:space:]]*\(#\|$\)' "$HERE/corpus.txt" | while read -r name url commit root patterns; do
+  src="$CACHE/src/$name"
+  if [ -d "$src/.git" ] && [ "$(git -C "$src" rev-parse HEAD 2>/dev/null)" = "$commit" ]; then
+    echo "cached   $name @ ${commit:0:12}"
+  else
+    echo "fetching $name @ ${commit:0:12} from $url"
+    rm -rf "$src"
+    git init -q "$src"
+    git -C "$src" remote add origin "$url"
+    git -C "$src" config advice.detachedHead false
+    if [ -n "$patterns" ]; then
+      git -C "$src" config core.sparseCheckout true
+      # no-cone patterns (globs); `set -f` keeps the shell from expanding them
+      (set -f; printf '%s\n' $patterns) > "$src/.git/info/sparse-checkout"
+    fi
+    retry git -C "$src" fetch -q --depth 1 --filter=blob:none origin "$commit"
+    retry git -C "$src" checkout -q FETCH_HEAD
+    got="$(git -C "$src" rev-parse HEAD)"
+    [ "$got" = "$commit" ] || { echo "error: $name checked out $got, expected $commit" >&2; exit 1; }
+  fi
+  ln -sfn "$src/$root" "$CACHE/corpus/$name"
+done
+echo "corpus ready in $CACHE/corpus"

conformance/list_files.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+"""list_files.py <corpus_dir>: print the files the sweep probes, one per line,
+as <corpus>/<path> in byte order.
+
+  * every file named *.h5 *.hdf5 *.he5 *.nc *.nc4 *.hdf *.h5f in each corpus,
+    except netCDF classic / 64-bit-offset / CDF5 files (magic "CDF"): they are
+    not HDF5, so neither side can read them and they say nothing;
+  * plus, for cve_hdf5, every file in cvefiles/ and fuzzerfiles/ except
+    .md/.c sources — the reproducers are mostly extension-less, and they are
+    kept whatever their bytes look like (that is their point).
+"""
+import os
+import sys
+
+EXTS = (".h5", ".hdf5", ".he5", ".nc", ".nc4", ".hdf", ".h5f")
+
+
+def walk(top):
+    for dirpath, dirnames, filenames in os.walk(top):
+        dirnames[:] = [d for d in dirnames if d != ".git"]
+        for fn in filenames:
+            p = os.path.join(dirpath, fn)
+            if os.path.isfile(p) and not os.path.islink(p):
+                yield os.path.relpath(p, top)
+
+
+def main(root):
+    out = set()
+    for corpus in sorted(os.listdir(root)):
+        top = os.path.join(root, corpus)
+        if not os.path.isdir(top):
+            continue
+        for rel in walk(top):
+            path = os.path.join(top, rel)
+            if rel.lower().endswith(EXTS):
+                with open(path, "rb") as fh:
+                    if fh.read(3) == b"CDF":
+                        continue
+                out.add(f"{corpus}/{rel}")
+            elif corpus == "cve_hdf5" and rel.split(os.sep)[0] in ("cvefiles", "fuzzerfiles") \
+                    and not rel.endswith((".md", ".c")):
+                out.add(f"{corpus}/{rel}")
+    for f in sorted(out, key=lambda s: s.encode()):
+        print(f)
+
+
+if __name__ == "__main__":
+    main(sys.argv[1])

conformance/probe/Cargo.lock

@@ -0,0 +1,491 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 4
+
+[[package]]
+name = "adler2"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
+
+[[package]]
+name = "better_io"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ef0a3155e943e341e557863e69a708999c94ede624e37865c8e2a91b94efa78f"
+
+[[package]]
+name = "block-buffer"
+version = "0.10.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
+dependencies = [
+ "generic-array",
+]
+
+[[package]]
+name = "byteorder"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
+
+[[package]]
+name = "bzip2"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f3a53fac24f34a81bc9954b5d6cfce0c21e18ec6959f44f56e8e90e4bb7c346c"
+dependencies = [
+ "libbz2-rs-sys",
+]
+
+[[package]]
+name = "cc"
+version = "1.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f360145194ee8e21db5ee7f3fcd4fe52210864c75c985dae33218202c8bbe040"
+dependencies = [
+ "find-msvc-tools",
+ "jobserver",
+ "libc",
+ "shlex",
+]
+
+[[package]]
+name = "cfg-if"
+version = "1.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600"
+
+[[package]]
+name = "clawhdf5-format"
+version = "2.7.0"
+dependencies = [
+ "byteorder",
+ "bzip2",
+ "flate2",
+ "libaec-sys",
+ "lz4_flex",
+ "pco",
+ "portable-atomic",
+ "ruzstd",
+ "sha2",
+ "snap",
+ "zstd",
+]
+
+[[package]]
+name = "conformance-probe"
+version = "0.1.0"
+dependencies = [
+ "clawhdf5-format",
+ "serde_json",
+ "sha2",
+]
+
+[[package]]
+name = "cpufeatures"
+version = "0.2.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "crc32fast"
+version = "1.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78"
+dependencies = [
+ "cfg-if",
+]
+
+[[package]]
+name = "crunchy"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5"
+
+[[package]]
+name = "crypto-common"
+version = "0.1.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
+dependencies = [
+ "generic-array",
+ "typenum",
+]
+
+[[package]]
+name = "digest"
+version = "0.10.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
+dependencies = [
+ "block-buffer",
+ "crypto-common",
+]
+
+[[package]]
+name = "dtype_dispatch"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ab23e69df104e2fd85ee63a533a22d2132ef5975dc6b36f9f3e5a7305e4a8ed7"
+
+[[package]]
+name = "find-msvc-tools"
+version = "0.1.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484"
+
+[[package]]
+name = "flate2"
+version = "1.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb"
+dependencies = [
+ "crc32fast",
+ "miniz_oxide",
+ "zlib-rs",
+]
+
+[[package]]
+name = "generic-array"
+version = "0.14.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
+dependencies = [
+ "typenum",
+ "version_check",
+]
+
+[[package]]
+name = "getrandom"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "r-efi",
+]
+
+[[package]]
+name = "half"
+version = "2.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b"
+dependencies = [
+ "cfg-if",
+ "crunchy",
+ "zerocopy",
+]
+
+[[package]]
+name = "itoa"
+version = "1.0.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
+
+[[package]]
+name = "jobserver"
+version = "0.1.35"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3"
+dependencies = [
+ "getrandom",
+ "libc",
+]
+
+[[package]]
+name = "libaec-sys"
+version = "0.1.0"
+dependencies = [
+ "pkg-config",
+]
+
+[[package]]
+name = "libbz2-rs-sys"
+version = "0.2.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "34b357333733e8260735ba5894eb928c02ecc69c78715f01a8019e7fa7f2db4c"
+
+[[package]]
+name = "libc"
+version = "0.2.189"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2"
+
+[[package]]
+name = "lz4_flex"
+version = "0.11.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "373f5eceeeab7925e0c1098212f2fbc4d416adec9d35051a6ab251e824c1854a"
+dependencies = [
+ "twox-hash",
+]
+
+[[package]]
+name = "memchr"
+version = "2.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
+
+[[package]]
+name = "miniz_oxide"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c"
+dependencies = [
+ "adler2",
+ "simd-adler32",
+]
+
+[[package]]
+name = "pco"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "386342cad4c6e97f081568e5d910ea7d871314c843aa8fc564f2a6b64cab9456"
+dependencies = [
+ "better_io",
+ "dtype_dispatch",
+ "half",
+ "rand_xoshiro",
+]
+
+[[package]]
+name = "pkg-config"
+version = "0.3.34"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548"
+
+[[package]]
+name = "portable-atomic"
+version = "1.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85"
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.107"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.47"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "r-efi"
+version = "6.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"
+
+[[package]]
+name = "rand_core"
+version = "0.6.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
+
+[[package]]
+name = "rand_xoshiro"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6f97cdb2a36ed4183de61b2f824cc45c9f1037f28afe0a322e9fff4c108b5aaa"
+dependencies = [
+ "rand_core",
+]
+
+[[package]]
+name = "ruzstd"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a252f5e20f038fe7b4ea53e073e65398d652c864cc162fc77c56c2f13717b888"
+dependencies = [
+ "twox-hash",
+]
+
+[[package]]
+name = "serde"
+version = "1.0.229"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
+dependencies = [
+ "serde_core",
+]
+
+[[package]]
+name = "serde_core"
+version = "1.0.229"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.229"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 3.0.6",
+]
+
+[[package]]
+name = "serde_json"
+version = "1.0.151"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14"
+dependencies = [
+ "itoa",
+ "memchr",
+ "serde",
+ "serde_core",
+ "zmij",
+]
+
+[[package]]
+name = "sha2"
+version = "0.10.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
+dependencies = [
+ "cfg-if",
+ "cpufeatures",
+ "digest",
+]
+
+[[package]]
+name = "shlex"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"
+
+[[package]]
+name = "simd-adler32"
+version = "0.3.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea"
+
+[[package]]
+name = "snap"
+version = "1.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "199905e6153d6405f9728fe44daace35f8f837bbf830bb6e85fbd5828709a886"
+
+[[package]]
+name = "syn"
+version = "2.0.119"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "syn"
+version = "3.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "twox-hash"
+version = "2.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5283634e518fe9e82c7b20520bb4bc209009fd16c82077c802f8111ecbb0117a"
+
+[[package]]
+name = "typenum"
+version = "1.20.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20"
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954"
+
+[[package]]
+name = "version_check"
+version = "0.9.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
+
+[[package]]
+name = "zerocopy"
+version = "0.8.59"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6df92bf3d9227be3d53173901ddbffac2babc27ae50f397776ffd6dc33f800cb"
+dependencies = [
+ "zerocopy-derive",
+]
+
+[[package]]
+name = "zerocopy-derive"
+version = "0.8.59"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac4f328cf2f05d084e496c3e9c3f33ed0a183656a16e1fcec4d464d8373aec82"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.119",
+]
+
+[[package]]
+name = "zlib-rs"
+version = "0.6.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112"
+
+[[package]]
+name = "zmij"
+version = "1.0.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
+
+[[package]]
+name = "zstd"
+version = "0.13.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a"
+dependencies = [
+ "zstd-safe",
+]
+
+[[package]]
+name = "zstd-safe"
+version = "7.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "64d80649ab6db9d9f6f9c80a40becd948eda4714a0a5ac8c4d157a32231c7882"
+dependencies = [
+ "zstd-sys",
+]
+
+[[package]]
+name = "zstd-sys"
+version = "2.1.0+zstd.1.5.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0ef0a8027ec3ee71300ab3bcbcd0393f434aa72b91ca6d635a39941deae8eea0"
+dependencies = [
+ "cc",
+ "pkg-config",
+]

conformance/probe/Cargo.toml

@@ -0,0 +1,25 @@
+[package]
+name = "conformance-probe"
+version = "0.1.0"
+edition = "2024"
+rust-version = "1.92"
+publish = false
+description = "Walks an HDF5 file with clawhdf5-format and prints a canonical JSON description (see conformance/README.md)"
+
+# Deliberately outside the main workspace: `cargo test --workspace` never
+# builds it, and it links the optional C codecs (zstd, libaec) that the core
+# crates' default build must not.
+[workspace]
+
+[dependencies]
+clawhdf5-format = { path = "../../crates/clawhdf5-format", features = ["lz4", "zstd", "szip", "pcodec", "plugin-filters"] }
+serde_json = "1"
+sha2 = "0.10"
+
+[profile.release]
+# Keep panics catchable (the probe records them per object) and turn integer
+# overflow into a reported panic instead of silent wraparound.
+debug = 1
+overflow-checks = true
+debug-assertions = true
+panic = "unwind"

conformance/probe/src/main.rs

@@ -0,0 +1,924 @@
+//! Conformance probe: walks an HDF5 file with clawhdf5-format (the same calls
+//! the `clawhdf5` facade makes) and prints a canonical JSON description:
+//! every hard-linked object (sorted-name DFS, deduplicated by header address),
+//! and for each dataset / attribute its shape plus the SHA-256 of its values
+//! in a canonical encoding shared with `ref.py`.
+//!
+//! Canonical value encoding (per element, concatenated, row-major):
+//!   int / float / bitfield / enum / time : element bytes, little-endian
+//!   non-IEEE-layout float (e.g. N-Bit)   : the IEEE float of the same size it converts to
+//!   int with bit offset / short precision: the full-width integer it converts to
+//!   opaque                               : raw bytes
+//!   compound                             : members in declaration order (padding dropped)
+//!   array                                : base elements row-major
+//!   string (fixed or VL)                 : b'S' + u32le len + bytes (cut at first NUL, trailing spaces stripped)
+//!   VL sequence                          : b'V' + u32le count + base elements
+//!   reference                            : b'R' (payload not compared)
+//!
+//! Every object is processed inside catch_unwind; a caught panic is recorded
+//! with its message, location and the clawhdf5 frames of its backtrace.
+
+use std::cell::RefCell;
+use std::collections::{HashMap, HashSet};
+use std::panic::{self, AssertUnwindSafe};
+use std::rc::Rc;
+
+use clawhdf5_format::attribute::extract_attributes_full;
+use clawhdf5_format::data_layout::DataLayout;
+use clawhdf5_format::data_read;
+use clawhdf5_format::dataspace::{Dataspace, DataspaceType};
+use clawhdf5_format::datatype::{Datatype, DatatypeByteOrder};
+use clawhdf5_format::filter_pipeline::FilterPipeline;
+use clawhdf5_format::global_heap::GlobalHeapCollection;
+use clawhdf5_format::group_v1::{self, GroupEntry};
+use clawhdf5_format::group_v2;
+use clawhdf5_format::message_type::MessageType;
+use clawhdf5_format::object_header::ObjectHeader;
+use clawhdf5_format::signature;
+use clawhdf5_format::superblock::Superblock;
+use clawhdf5_format::symbol_table::SymbolTableMessage;
+use serde_json::{Map, Value, json};
+use sha2::{Digest, Sha256};
+
+const MAX_BYTES: u64 = 200 * 1024 * 1024;
+const MAX_OBJECTS: usize = 200_000;
+
+thread_local! {
+    static LAST_PANIC: RefCell<Option<String>> = const { RefCell::new(None) };
+}
+
+fn install_hook() {
+    panic::set_hook(Box::new(|info| {
+        let msg = if let Some(s) = info.payload().downcast_ref::<&str>() {
+            s.to_string()
+        } else if let Some(s) = info.payload().downcast_ref::<String>() {
+            s.clone()
+        } else {
+            "<non-string panic>".into()
+        };
+        let loc = info
+            .location()
+            .map(|l| format!("{}:{}", l.file(), l.line()))
+            .unwrap_or_default();
+        let bt = std::backtrace::Backtrace::force_capture().to_string();
+        // keep only frames from clawhdf5 code
+        let mut frames = Vec::new();
+        let lines: Vec<&str> = bt.lines().collect();
+        for (i, l) in lines.iter().enumerate() {
+            let t = l.trim();
+            if t.contains("clawhdf5_format::") || t.contains("conformance_probe::") {
+                let at = lines
+                    .get(i + 1)
+                    .map(|n| n.trim())
+                    .filter(|n| n.starts_with("at "))
+                    .map(|n| {
+                        let n = n.trim_start_matches("at ");
+                        match n.find("/crates/") {
+                            Some(p) => n[p + 1..].to_string(),
+                            None => n.to_string(),
+                        }
+                    })
+                    .unwrap_or_default();
+                let name = t.split_once(": ").map(|x| x.1).unwrap_or(t);
+                frames.push(format!("{name} ({at})"));
+                if frames.len() >= 12 {
+                    break;
+                }
+            }
+        }
+        let full = format!("PANIC: {msg} @ {loc}\n  {}", frames.join("\n  "));
+        eprintln!("{full}");
+        LAST_PANIC.with(|p| *p.borrow_mut() = Some(full));
+    }));
+}
+
+/// Run `f`, turning a panic into Err("PANIC: ...").
+fn guarded<T>(f: impl FnOnce() -> Result<T, String>) -> Result<T, String> {
+    match panic::catch_unwind(AssertUnwindSafe(f)) {
+        Ok(r) => r,
+        Err(_) => Err(LAST_PANIC
+            .with(|p| p.borrow_mut().take())
+            .unwrap_or_else(|| "PANIC: <unknown>".into())),
+    }
+}
+
+fn e<E: std::fmt::Debug>(x: E) -> String {
+    format!("{x:?}")
+}
+
+struct Ctx<'a> {
+    data: &'a [u8],
+    os: u8,
+    ls: u8,
+    base_dir: std::path::PathBuf,
+    heaps: RefCell<HashMap<u64, Result<Rc<GlobalHeapCollection>, String>>>,
+}
+
+impl<'a> Ctx<'a> {
+    fn header(&self, addr: u64) -> Result<ObjectHeader, String> {
+        ObjectHeader::parse(self.data, addr as usize, self.os, self.ls).map_err(e)
+    }
+
+    fn payload(&self, h: &ObjectHeader, t: MessageType) -> Result<Option<Vec<u8>>, String> {
+        match h.messages.iter().find(|m| m.msg_type == t) {
+            None => Ok(None),
+            Some(m) => {
+                clawhdf5_format::shared_message::message_data(self.data, m, self.os, self.ls)
+                    .map(|c| Some(c.into_owned()))
+                    .map_err(e)
+            }
+        }
+    }
+
+    fn heap_obj(&self, addr: u64, idx: u32) -> Result<Vec<u8>, String> {
+        let coll = {
+            let mut cache = self.heaps.borrow_mut();
+            cache
+                .entry(addr)
+                .or_insert_with(|| {
+                    GlobalHeapCollection::parse(self.data, addr as usize, self.ls)
+                        .map(Rc::new)
+                        .map_err(e)
+                })
+                .clone()?
+        };
+        coll.get_object(idx as u16)
+            .map(|o| o.data.clone())
+            .ok_or_else(|| {
+                format!("GlobalHeapObjectNotFound {{ collection_address: {addr}, index: {idx} }}")
+            })
+    }
+
+    fn read_offset(&self, b: &[u8]) -> u64 {
+        let mut v = 0u64;
+        for (i, x) in b.iter().take(self.os as usize).enumerate() {
+            v |= (*x as u64) << (8 * i);
+        }
+        v
+    }
+
+    fn canon(&self, dt: &Datatype, b: &[u8], out: &mut Vec<u8>) -> Result<(), String> {
+        let size = dt.type_size() as usize;
+        if b.len() < size {
+            return Err(format!(
+                "canon: element slice {} < type size {size}",
+                b.len()
+            ));
+        }
+        match dt {
+            Datatype::FloatingPoint { .. } if !ieee_layout(dt) => {
+                canon_custom_float(dt, &b[..size], out)?
+            }
+            Datatype::FixedPoint { .. } if partial_int(dt) => {
+                canon_partial_int(dt, &b[..size], out)?
+            }
+            Datatype::FixedPoint { byte_order, .. }
+            | Datatype::BitField { byte_order, .. }
+            | Datatype::FloatingPoint { byte_order, .. } => match byte_order {
+                DatatypeByteOrder::LittleEndian => out.extend_from_slice(&b[..size]),
+                DatatypeByteOrder::BigEndian => out.extend(b[..size].iter().rev()),
+                DatatypeByteOrder::Vax => return Err("canon: VAX byte order".into()),
+            },
+            Datatype::Time { .. } | Datatype::Opaque { .. } => out.extend_from_slice(&b[..size]),
+            Datatype::String { .. } => canon_str(&b[..size], out),
+            Datatype::Compound { members, .. } => {
+                for m in members {
+                    let off = m.byte_offset as usize;
+                    let ms = m.datatype.type_size() as usize;
+                    if off.checked_add(ms).is_none_or(|end| end > size) {
+                        return Err(format!("canon: member {} out of bounds", m.name));
+                    }
+                    self.canon(&m.datatype, &b[off..off + ms], out)?;
+                }
+            }
+            Datatype::Reference { .. } => out.push(b'R'),
+            Datatype::Enumeration { base_type, .. } => self.canon(base_type, b, out)?,
+            Datatype::Array {
+                base_type,
+                dimensions,
+            } => {
+                let n: usize = dimensions.iter().map(|d| *d as usize).product();
+                let bs = base_type.type_size() as usize;
+                for i in 0..n {
+                    self.canon(base_type, &b[i * bs..], out)?;
+                }
+            }
+            Datatype::VariableLength {
+                is_string,
+                base_type,
+                ..
+            } => {
+                let len = u32::from_le_bytes([b[0], b[1], b[2], b[3]]) as usize;
+                let addr = self.read_offset(&b[4..]);
+                let idx_off = 4 + self.os as usize;
+                let idx = u32::from_le_bytes([
+                    b[idx_off],
+                    b[idx_off + 1],
+                    b[idx_off + 2],
+                    b[idx_off + 3],
+                ]);
+                let obj = if len == 0 || addr == 0 || addr == u64::MAX >> (64 - 8 * self.os as u32)
+                {
+                    Vec::new()
+                } else {
+                    self.heap_obj(addr, idx)?
+                };
+                if *is_string {
+                    let l = len.min(obj.len());
+                    canon_str(&obj[..l], out);
+                } else {
+                    let bs = base_type.type_size() as usize;
+                    if bs == 0 {
+                        return Err("canon: VL base size 0".into());
+                    }
+                    let need = len.checked_mul(bs).ok_or("canon: VL overflow")?;
+                    if len > 0 && obj.len() < need {
+                        return Err(format!("canon: VL object {} < {need}", obj.len()));
+                    }
+                    out.push(b'V');
+                    out.extend_from_slice(&(len as u32).to_le_bytes());
+                    for i in 0..len {
+                        self.canon(base_type, &obj[i * bs..], out)?;
+                    }
+                }
+            }
+        }
+        Ok(())
+    }
+
+    /// Returns (shape json, n_elements)
+    fn shape(ds: &Dataspace) -> (Value, u64) {
+        match ds.space_type {
+            DataspaceType::Null => (Value::String("null".into()), 0),
+            DataspaceType::Scalar => (json!([]), 1),
+            DataspaceType::Simple => {
+                let n = ds.dimensions.iter().fold(1u64, |a, d| a.saturating_mul(*d));
+                (json!(ds.dimensions), n)
+            }
+        }
+    }
+
+    fn hash_values(
+        &self,
+        dt: &Datatype,
+        raw: &[u8],
+        n: u64,
+        rec: &mut Map<String, Value>,
+    ) -> Result<(), String> {
+        let size = dt.type_size() as usize;
+        let need = (n as usize).checked_mul(size).ok_or("n*size overflow")?;
+        if raw.len() != need {
+            return Err(format!(
+                "raw length {} != n_elements {n} * type_size {size}",
+                raw.len()
+            ));
+        }
+        let mut canon = Vec::with_capacity(need);
+        for i in 0..n as usize {
+            self.canon(dt, &raw[i * size..(i + 1) * size], &mut canon)?;
+        }
+        let h = Sha256::digest(&canon);
+        rec.insert("hash".into(), Value::String(hex(&h)));
+        rec.insert(
+            "head".into(),
+            Value::String(hex(&canon[..canon.len().min(48)])),
+        );
+        Ok(())
+    }
+
+    /// VDS source files resolve next to the virtual file; like the library,
+    /// refuse absolute paths and `..`.
+    fn vds_resolver(
+        &self,
+    ) -> impl Fn(&str) -> Result<Option<Vec<u8>>, clawhdf5_format::error::FormatError> + use<> {
+        let base = self.base_dir.clone();
+        move |name: &str| {
+            use clawhdf5_format::error::FormatError;
+            let p = std::path::Path::new(name);
+            if p.is_absolute()
+                || p.components()
+                    .any(|c| matches!(c, std::path::Component::ParentDir))
+            {
+                return Err(FormatError::ChunkedReadError(format!("refused {name}")));
+            }
+            match std::fs::read(base.join(p)) {
+                Ok(b) => Ok(Some(b)),
+                Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(None),
+                Err(err) => Err(FormatError::ChunkedReadError(err.to_string())),
+            }
+        }
+    }
+
+    fn read_named_datatype(&self, h: &ObjectHeader) -> Result<(), String> {
+        let dtb = self
+            .payload(h, MessageType::Datatype)?
+            .ok_or("MissingMessage(Datatype)")?;
+        Datatype::parse_in_header(&dtb, h.version).map_err(e)?;
+        Ok(())
+    }
+
+    fn read_dataset(&self, h: &ObjectHeader, rec: &mut Map<String, Value>) -> Result<(), String> {
+        let dtb = self
+            .payload(h, MessageType::Datatype)?
+            .ok_or("MissingMessage(Datatype)")?;
+        let (dt, _) = Datatype::parse_in_header(&dtb, h.version).map_err(e)?;
+        rec.insert("dtype".into(), Value::String(dtype_str(&dt)));
+        let dsb = self
+            .payload(h, MessageType::Dataspace)?
+            .ok_or("MissingMessage(Dataspace)")?;
+        let mut ds = Dataspace::parse(&dsb, self.ls).map_err(e)?;
+        // A virtual dataset's extent can come from its sources (unlimited /
+        // printf mappings), as h5py reports it, rather than the stored one.
+        if let Some(lm) = h
+            .messages
+            .iter()
+            .find(|m| m.msg_type == MessageType::DataLayout)
+            && let Ok(dl @ DataLayout::Virtual { .. }) =
+                DataLayout::parse(&lm.data, self.os, self.ls)
+        {
+            let resolver = self.vds_resolver();
+            ds.dimensions = clawhdf5_format::vds::virtual_dataset_extent(
+                self.data,
+                &dl,
+                &ds,
+                self.os,
+                self.ls,
+                Some(&resolver),
+            )
+            .map_err(e)?;
+        }
+        let (shape, n) = Self::shape(&ds);
+        rec.insert("shape".into(), shape);
+        if n.saturating_mul(dt.type_size() as u64) > MAX_BYTES {
+            rec.insert("skipped".into(), Value::String("too large".into()));
+            return Ok(());
+        }
+        let lm = h
+            .messages
+            .iter()
+            .find(|m| m.msg_type == MessageType::DataLayout)
+            .ok_or("MissingMessage(DataLayout)")?;
+        let dl = DataLayout::parse(&lm.data, self.os, self.ls).map_err(e)?;
+        rec.insert(
+            "layout".into(),
+            Value::String(
+                match &dl {
+                    DataLayout::Compact { .. } => "compact",
+                    DataLayout::Contiguous { .. } => "contiguous",
+                    DataLayout::Chunked { .. } => "chunked",
+                    DataLayout::Virtual { .. } => "virtual",
+                }
+                .into(),
+            ),
+        );
+        let pipeline = match self.payload(h, MessageType::FilterPipeline)? {
+            Some(p) => Some(FilterPipeline::parse(&p).map_err(e)?),
+            None => None,
+        };
+        if let Some(p) = &pipeline {
+            rec.insert(
+                "filters".into(),
+                json!(p.filters.iter().map(|f| f.filter_id).collect::<Vec<_>>()),
+            );
+        }
+        let raw = if matches!(dl, DataLayout::Virtual { .. }) {
+            let resolver = self.vds_resolver();
+            let fill = clawhdf5_format::fill_value::dataset_fill_value_in(
+                self.data,
+                &h.messages,
+                self.os,
+                self.ls,
+            )
+            .map_err(e)?;
+            clawhdf5_format::vds::read_virtual_dataset(
+                self.data,
+                &dl,
+                &ds,
+                &dt,
+                fill.as_deref(),
+                self.os,
+                self.ls,
+                Some(&resolver),
+            )
+            .map_err(e)?
+            .data
+        } else {
+            let cache = clawhdf5_format::chunk_cache::ChunkCache::new();
+            clawhdf5_format::fill_value::read_full_with_fill::<clawhdf5_format::error::FormatError>(
+                &h.messages,
+                self.data,
+                &dl,
+                &ds,
+                dt.type_size() as usize,
+                self.os,
+                self.ls,
+                || {
+                    data_read::read_raw_data_cached(
+                        self.data,
+                        &dl,
+                        &ds,
+                        &dt,
+                        pipeline.as_ref(),
+                        self.os,
+                        self.ls,
+                        &cache,
+                    )
+                },
+            )
+            .map_err(e)?
+        };
+        self.hash_values(&dt, &raw, n, rec)
+    }
+
+    fn attrs(&self, h: &ObjectHeader) -> Result<Map<String, Value>, String> {
+        let msgs = extract_attributes_full(self.data, h, self.os, self.ls).map_err(e)?;
+        let mut out = Map::new();
+        for a in &msgs {
+            let r = guarded(|| {
+                let mut rec = Map::new();
+                rec.insert("dtype".into(), Value::String(dtype_str(&a.datatype)));
+                let (shape, n) = Self::shape(&a.dataspace);
+                rec.insert("shape".into(), shape);
+                self.hash_values(&a.datatype, &a.raw_data, n, &mut rec)?;
+                Ok(rec)
+            });
+            let v = match r {
+                Ok(rec) => Value::Object(rec),
+                Err(msg) => json!({ "error": msg }),
+            };
+            out.insert(a.name.clone(), v);
+        }
+        Ok(out)
+    }
+
+    fn entries(&self, h: &ObjectHeader) -> Result<Vec<GroupEntry>, String> {
+        let v1 = h
+            .messages
+            .iter()
+            .find(|m| m.msg_type == MessageType::SymbolTable);
+        if let Some(m) = v1 {
+            let stm = SymbolTableMessage::parse(&m.data, self.os).map_err(e)?;
+            group_v1::resolve_v1_group_entries(self.data, &stm, self.os, self.ls).map_err(e)
+        } else if h
+            .messages
+            .iter()
+            .any(|m| m.msg_type == MessageType::LinkInfo || m.msg_type == MessageType::Link)
+        {
+            group_v2::resolve_v2_group_entries(self.data, h, self.os, self.ls).map_err(e)
+        } else {
+            Ok(Vec::new())
+        }
+    }
+}
+
+/// Element bytes as an unsigned integer (at most 16 bytes), honouring byte order.
+fn element_bits(b: &[u8], byte_order: &DatatypeByteOrder) -> Result<u128, String> {
+    if b.len() > 16 {
+        return Err(format!("canon: {}-byte numeric element", b.len()));
+    }
+    let mut v = 0u128;
+    match byte_order {
+        DatatypeByteOrder::LittleEndian => {
+            for (i, x) in b.iter().enumerate() {
+                v |= u128::from(*x) << (8 * i);
+            }
+        }
+        DatatypeByteOrder::BigEndian => {
+            for x in b {
+                v = (v << 8) | u128::from(*x);
+            }
+        }
+        DatatypeByteOrder::Vax => return Err("canon: VAX byte order".into()),
+    }
+    Ok(v)
+}
+
+fn field(v: u128, pos: u32, len: u32) -> u128 {
+    if len == 0 || pos >= 128 {
+        return 0;
+    }
+    let v = v >> pos;
+    if len >= 128 {
+        v
+    } else {
+        v & ((1u128 << len) - 1)
+    }
+}
+
+/// True when a float's bit fields are exactly IEEE 754 binary16/32/64 for its
+/// size. h5py hands back such a type's bytes untouched; any other layout (an
+/// N-Bit `H5Tset_precision` float, say) is *converted* by libhdf5 into the
+/// numpy float of the same size, so comparing raw bytes would be meaningless.
+fn ieee_layout(dt: &Datatype) -> bool {
+    let Datatype::FloatingPoint {
+        size,
+        bit_offset,
+        bit_precision,
+        exponent_location,
+        exponent_size,
+        mantissa_location,
+        mantissa_size,
+        exponent_bias,
+        ..
+    } = dt
+    else {
+        return true;
+    };
+    let std = match size {
+        2 => (16, 10, 5, 10, 15),
+        4 => (32, 23, 8, 23, 127),
+        8 => (64, 52, 11, 52, 1023),
+        _ => return true, // no same-size numpy float to convert to: compare raw
+    };
+    *bit_offset == 0
+        && (
+            *bit_precision,
+            *exponent_location,
+            *exponent_size,
+            *mantissa_size,
+            *exponent_bias,
+        ) == (std.0, std.1, std.2, std.3, std.4)
+        && *mantissa_location == 0
+}
+
+/// Canonicalise a non-IEEE-layout float the way libhdf5's float->float
+/// conversion presents it to h5py: as the IEEE float of the same size.
+/// Assumes the implied-leading-one normalisation and the sign bit at the top
+/// of the precision (what `H5Tset_precision` produces; the parser does not
+/// keep either field).
+fn canon_custom_float(dt: &Datatype, b: &[u8], out: &mut Vec<u8>) -> Result<(), String> {
+    let Datatype::FloatingPoint {
+        size,
+        byte_order,
+        bit_offset,
+        bit_precision,
+        exponent_location,
+        exponent_size,
+        mantissa_location,
+        mantissa_size,
+        exponent_bias,
+    } = dt
+    else {
+        unreachable!()
+    };
+    let (esize, msize) = (u32::from(*exponent_size), u32::from(*mantissa_size));
+    if esize == 0 || esize > 30 || msize > 64 {
+        return Err(format!("canon: unsupported float layout e{esize} m{msize}"));
+    }
+    let v = element_bits(b, byte_order)?;
+    let sign_pos = (u32::from(*bit_offset) + u32::from(*bit_precision)).saturating_sub(1);
+    let neg = field(v, sign_pos, 1) == 1;
+    let e = field(v, u32::from(*exponent_location), esize) as i64;
+    let m = field(v, u32::from(*mantissa_location), msize);
+    let emax = (1i64 << esize) - 1;
+    let bias = i64::from(*exponent_bias);
+    let mag = if e == emax {
+        if m == 0 { f64::INFINITY } else { f64::NAN }
+    } else if e == 0 {
+        (m as f64) * 2f64.powi((1 - bias - msize as i64) as i32)
+    } else {
+        ((1u128 << msize) as f64 + m as f64) * 2f64.powi((e - bias - msize as i64) as i32)
+    };
+    let x = if neg { -mag } else { mag };
+    match size {
+        2 => out
+            .extend_from_slice(&clawhdf5_format::float16::f32_to_f16_bits(x as f32).to_le_bytes()),
+        4 => out.extend_from_slice(&(x as f32).to_le_bytes()),
+        8 => out.extend_from_slice(&x.to_le_bytes()),
+        _ => unreachable!("ieee_layout keeps other sizes raw"),
+    }
+    Ok(())
+}
+
+/// Integers stored with a bit offset or reduced precision (N-Bit): libhdf5
+/// converts them to the full-width integer of the same size, shifting the
+/// value down and sign-extending from the top precision bit.
+fn canon_partial_int(dt: &Datatype, b: &[u8], out: &mut Vec<u8>) -> Result<(), String> {
+    let Datatype::FixedPoint {
+        size,
+        byte_order,
+        signed,
+        bit_offset,
+        bit_precision,
+    } = dt
+    else {
+        unreachable!()
+    };
+    let prec = u32::from(*bit_precision);
+    let v = element_bits(b, byte_order)?;
+    let mut x = field(v, u32::from(*bit_offset), prec);
+    if *signed && prec > 0 && prec < 128 && field(x, prec - 1, 1) == 1 {
+        x |= !0u128 << prec;
+    }
+    out.extend_from_slice(&x.to_le_bytes()[..*size as usize]);
+    Ok(())
+}
+
+fn partial_int(dt: &Datatype) -> bool {
+    matches!(dt, Datatype::FixedPoint { size, bit_offset, bit_precision, .. }
+        if *bit_offset != 0 || u32::from(*bit_precision) != size * 8)
+}
+
+fn canon_str(b: &[u8], out: &mut Vec<u8>) {
+    let cut = b.iter().position(|&c| c == 0).unwrap_or(b.len());
+    let mut s = &b[..cut];
+    while let [rest @ .., b' '] = s {
+        s = rest;
+    }
+    out.push(b'S');
+    out.extend_from_slice(&(s.len() as u32).to_le_bytes());
+    out.extend_from_slice(s);
+}
+
+fn hex(b: &[u8]) -> String {
+    b.iter().map(|x| format!("{x:02x}")).collect()
+}
+
+fn dtype_str(dt: &Datatype) -> String {
+    match dt {
+        Datatype::FixedPoint {
+            size,
+            signed,
+            byte_order,
+            ..
+        } => {
+            format!(
+                "{}{}{}",
+                bo(byte_order),
+                if *signed { "i" } else { "u" },
+                size
+            )
+        }
+        Datatype::FloatingPoint {
+            size, byte_order, ..
+        } => format!("{}f{}", bo(byte_order), size),
+        Datatype::BitField {
+            size, byte_order, ..
+        } => format!("{}b{}", bo(byte_order), size),
+        Datatype::Time { size, .. } => format!("time{size}"),
+        Datatype::String { size, .. } => format!("S{size}"),
+        Datatype::Opaque { size, .. } => format!("V{size}"),
+        Datatype::Compound { size, members } => format!(
+            "{{{}}}{size}",
+            members
+                .iter()
+                .map(|m| format!("{}:{}", m.name, dtype_str(&m.datatype)))
+                .collect::<Vec<_>>()
+                .join(",")
+        ),
+        Datatype::Reference { ref_type, .. } => format!("ref({ref_type:?})"),
+        Datatype::Enumeration { base_type, .. } => format!("enum({})", dtype_str(base_type)),
+        Datatype::VariableLength {
+            is_string: true, ..
+        } => "vlstr".into(),
+        Datatype::VariableLength { base_type, .. } => format!("vlen({})", dtype_str(base_type)),
+        Datatype::Array {
+            base_type,
+            dimensions,
+        } => format!("({}){dimensions:?}", dtype_str(base_type)),
+    }
+}
+
+fn bo(b: &DatatypeByteOrder) -> &'static str {
+    match b {
+        DatatypeByteOrder::LittleEndian => "<",
+        DatatypeByteOrder::BigEndian => ">",
+        DatatypeByteOrder::Vax => "vax",
+    }
+}
+
+fn is_group(h: &ObjectHeader) -> bool {
+    h.messages.iter().any(|m| {
+        matches!(
+            m.msg_type,
+            MessageType::LinkInfo | MessageType::Link | MessageType::SymbolTable
+        )
+    })
+}
+
+fn main() {
+    install_hook();
+    let path = std::env::args().nth(1).expect("usage: probe <file>");
+    let mut top = Map::new();
+    top.insert("file".into(), Value::String(path.clone()));
+    let data = match std::fs::read(&path) {
+        Ok(d) => d,
+        Err(err) => {
+            top.insert("open_error".into(), Value::String(format!("Io({err})")));
+            println!("{}", Value::Object(top));
+            return;
+        }
+    };
+    // Every address is relative to the superblock: look at the file from
+    // there on (past any user block), as libhdf5 does.
+    let hdf5: &[u8] = match signature::find_signature(&data) {
+        Ok(off) => &data[off..],
+        Err(_) => &data,
+    };
+    let sb = guarded(|| Superblock::parse(hdf5, 0).map_err(e));
+    let sb = match sb {
+        Ok(sb) => sb,
+        Err(msg) => {
+            top.insert("open_error".into(), Value::String(msg));
+            println!("{}", Value::Object(top));
+            return;
+        }
+    };
+    // libhdf5 refuses a truncated file and reads nothing past the recorded
+    // end of file.
+    let base = (data.len() - hdf5.len()) as u64;
+    let hdf5 = match sb.data_end(base, data.len() as u64) {
+        Ok(end) => &hdf5[..end as usize],
+        Err(err) => {
+            top.insert("open_error".into(), Value::String(e(err)));
+            println!("{}", Value::Object(top));
+            return;
+        }
+    };
+    top.insert("superblock_version".into(), json!(sb.version));
+    let ctx = Ctx {
+        data: hdf5,
+        os: sb.offset_size,
+        ls: sb.length_size,
+        base_dir: std::path::Path::new(&path)
+            .parent()
+            .map(|p| p.to_path_buf())
+            .unwrap_or_default(),
+        heaps: RefCell::new(HashMap::new()),
+    };
+    let mut objects: Vec<Value> = Vec::new();
+    let mut visited = HashSet::new();
+    let mut soft_v1 = 0u64;
+    // explicit DFS stack: (address, path)
+    let mut stack: Vec<(u64, String)> = vec![(sb.root_group_address, "/".to_string())];
+    while let Some((addr, p)) = stack.pop() {
+        if objects.len() >= MAX_OBJECTS {
+            top.insert("truncated".into(), json!(true));
+            break;
+        }
+        if !visited.insert(addr) {
+            continue;
+        }
+        let mut rec = Map::new();
+        rec.insert("path".into(), Value::String(p.clone()));
+        let r = guarded(|| {
+            let h = ctx.header(addr)?;
+            Ok(h)
+        });
+        let h = match r {
+            Ok(h) => h,
+            Err(msg) => {
+                rec.insert("kind".into(), Value::String("unknown".into()));
+                rec.insert("error".into(), Value::String(msg));
+                objects.push(Value::Object(rec));
+                continue;
+            }
+        };
+        let is_ds = h
+            .messages
+            .iter()
+            .any(|m| m.msg_type == MessageType::DataLayout);
+        let kind = if is_ds {
+            "dataset"
+        } else if is_group(&h) || addr == sb.root_group_address {
+            "group"
+        } else if h
+            .messages
+            .iter()
+            .any(|m| m.msg_type == MessageType::Datatype)
+        {
+            "datatype"
+        } else {
+            "unknown"
+        };
+        rec.insert("kind".into(), Value::String(kind.into()));
+        if kind == "dataset"
+            && let Err(msg) = guarded(|| ctx.read_dataset(&h, &mut rec))
+        {
+            rec.insert("error".into(), Value::String(msg));
+        }
+        // Opening a committed datatype decodes it (h5py's `f[name]` fails on
+        // one libhdf5 cannot decode), so decode it here too.
+        if kind == "datatype"
+            && let Err(msg) = guarded(|| ctx.read_named_datatype(&h))
+        {
+            rec.insert("error".into(), Value::String(msg));
+        }
+        if kind != "datatype" {
+            match guarded(|| ctx.attrs(&h)) {
+                Ok(m) => {
+                    rec.insert("attrs".into(), Value::Object(m));
+                }
+                Err(msg) => {
+                    rec.insert("attrs_error".into(), Value::String(msg));
+                }
+            }
+        }
+        if kind == "group" {
+            match guarded(|| ctx.entries(&h)) {
+                Ok(mut ents) => {
+                    ents.retain(|en| {
+                        if en.cache_type == 2 {
+                            soft_v1 += 1;
+                            false
+                        } else {
+                            true
+                        }
+                    });
+                    ents.sort_by(|a, b| a.name.cmp(&b.name));
+                    let base = if p == "/" { String::new() } else { p.clone() };
+                    for en in ents.into_iter().rev() {
+                        stack.push((en.object_header_address, format!("{base}/{}", en.name)));
+                    }
+                }
+                Err(msg) => {
+                    rec.insert("list_error".into(), Value::String(msg));
+                }
+            }
+        }
+        objects.push(Value::Object(rec));
+    }
+    if soft_v1 > 0 {
+        top.insert("v1_soft_link_entries".into(), json!(soft_v1));
+    }
+    top.insert("objects".into(), Value::Array(objects));
+    println!("{}", Value::Object(top));
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    /// The N-Bit float of libhdf5's `test/testfiles/le_data.h5`
+    /// (`Nbit_float_data_le`): offset 7, precision 20, sign bit 26, exponent
+    /// 20+6 (bias 31), mantissa 7+13.
+    fn nbit_f32(byte_order: DatatypeByteOrder) -> Datatype {
+        Datatype::FloatingPoint {
+            size: 4,
+            byte_order,
+            bit_offset: 7,
+            bit_precision: 20,
+            exponent_location: 20,
+            exponent_size: 6,
+            mantissa_location: 7,
+            mantissa_size: 13,
+            exponent_bias: 31,
+        }
+    }
+
+    fn canon_one(dt: &Datatype, bytes: &[u8]) -> Vec<u8> {
+        let mut out = Vec::new();
+        canon_custom_float(dt, bytes, &mut out).unwrap();
+        out
+    }
+
+    #[test]
+    fn nbit_float_canonicalises_to_the_value_libhdf5_returns() {
+        let le = nbit_f32(DatatypeByteOrder::LittleEndian);
+        let be = nbit_f32(DatatypeByteOrder::BigEndian);
+        assert!(!ieee_layout(&le));
+        // 1.0: exponent = bias, mantissa 0
+        let one: u32 = 31 << 20;
+        assert_eq!(canon_one(&le, &one.to_le_bytes()), 1.0f32.to_le_bytes());
+        assert_eq!(canon_one(&be, &one.to_be_bytes()), 1.0f32.to_le_bytes());
+        // -2.1999512 (h5py's reading of the file's -2.2): sign, e = 32, m = 819
+        let v: u32 = (1 << 26) | (32 << 20) | (819 << 7);
+        assert_eq!(
+            canon_one(&le, &v.to_le_bytes()),
+            (-2.199_951_2f32).to_le_bytes()
+        );
+        assert_eq!(canon_one(&le, &[0; 4]), 0.0f32.to_le_bytes());
+    }
+
+    #[test]
+    fn ieee_floats_keep_their_raw_bytes() {
+        let f32le = Datatype::FloatingPoint {
+            size: 4,
+            byte_order: DatatypeByteOrder::LittleEndian,
+            bit_offset: 0,
+            bit_precision: 32,
+            exponent_location: 23,
+            exponent_size: 8,
+            mantissa_location: 0,
+            mantissa_size: 23,
+            exponent_bias: 127,
+        };
+        assert!(ieee_layout(&f32le));
+    }
+
+    #[test]
+    fn partial_precision_int_is_shifted_and_sign_extended() {
+        let dt = Datatype::FixedPoint {
+            size: 4,
+            byte_order: DatatypeByteOrder::BigEndian,
+            signed: true,
+            bit_offset: 4,
+            bit_precision: 17,
+        };
+        assert!(partial_int(&dt));
+        let stored = (((-5i32) as u32) & 0x1_FFFF) << 4;
+        let mut out = Vec::new();
+        canon_partial_int(&dt, &stored.to_be_bytes(), &mut out).unwrap();
+        assert_eq!(out, (-5i32).to_le_bytes());
+    }
+}

conformance/ref.py

@@ -0,0 +1,259 @@
+#!/usr/bin/env python3
+"""Reference probe: same JSON as the Rust `conformance-probe`, produced with h5py.
+
+Walk: iterative DFS from '/', children in sorted (UTF-8 byte) name order, hard
+links only, each object once (first path wins, deduplicated by object identity).
+Canonical value encoding: see harness/src/main.rs.
+"""
+import hashlib
+import json
+import os
+import struct
+import sys
+
+import numpy as np
+import h5py
+
+try:
+    import hdf5plugin  # noqa: F401  registers blosc/lz4/zstd/bzip2/... filters
+except Exception:  # pragma: no cover
+    pass
+
+MAX_BYTES = 200 * 1024 * 1024
+MAX_OBJECTS = 200_000
+
+
+def canon_str(b, out):
+    if isinstance(b, str):
+        b = b.encode("utf-8", "surrogateescape")
+    b = bytes(b)
+    cut = b.find(b"\x00")
+    if cut >= 0:
+        b = b[:cut]
+    b = b.rstrip(b" ")
+    out += b"S" + struct.pack("<I", len(b)) + b
+
+
+def simple(dt):
+    if dt.fields:
+        return all(simple(dt.fields[n][0]) for n in dt.names)
+    if dt.subdtype:
+        return simple(dt.subdtype[0])
+    return dt.kind in "iufcbV"
+
+
+def packed(dt):
+    if dt.fields:
+        return np.dtype([(n, packed(dt.fields[n][0])) for n in dt.names])
+    if dt.subdtype:
+        base, shape = dt.subdtype
+        return np.dtype((packed(base), shape))
+    if dt.kind in "iufcb":
+        return dt.newbyteorder("<")
+    return dt
+
+
+def canon_el(dt, val, out):
+    if dt.fields:
+        for n in dt.names:
+            canon_el(dt.fields[n][0], val[n], out)
+        return
+    if dt.subdtype:
+        base, _ = dt.subdtype
+        for x in np.asarray(val).reshape(-1):
+            canon_el(base, x, out)
+        return
+    k = dt.kind
+    if k in "iufcb":
+        out += np.asarray(val, dtype=dt).astype(dt.newbyteorder("<")).tobytes()
+    elif k == "V":
+        out += np.asarray(val, dtype=dt).tobytes()
+    elif k == "S":
+        canon_str(val, out)
+    elif k == "O":
+        if h5py.check_string_dtype(dt) is not None:
+            canon_str(val if val is not None else b"", out)
+        elif h5py.check_ref_dtype(dt) is not None:
+            out += b"R"
+        else:
+            base = h5py.check_vlen_dtype(dt)
+            if base is None:
+                raise TypeError(f"unhandled object dtype {dt!r}")
+            arr = np.asarray(val if val is not None else [], dtype=base).reshape(-1)
+            out += b"V" + struct.pack("<I", arr.shape[0])
+            if simple(base):
+                out += arr.astype(packed(base)).tobytes()
+            else:
+                for x in arr:
+                    canon_el(base, x, out)
+    elif k == "U":
+        canon_str(str(val), out)
+    else:
+        raise TypeError(f"unhandled dtype kind {k} ({dt!r})")
+
+
+def has_obj(dt):
+    if dt.fields:
+        return any(has_obj(dt.fields[n][0]) for n in dt.names)
+    if dt.subdtype:
+        return has_obj(dt.subdtype[0])
+    return dt.kind == "O"
+
+
+def note_conversion(tid, dt, rec):
+    """h5py converts some file types (FP8, bfloat16, x87 long double, ...) to a
+    different-sized numpy type; then value bytes are not comparable."""
+    try:
+        if not has_obj(dt) and tid.get_size() != dt.itemsize:
+            rec["converted"] = f"file type size {tid.get_size()} -> numpy {dt} ({dt.itemsize})"
+    except Exception:  # noqa: BLE001
+        pass
+
+
+def hash_values(arr, dt, rec):
+    if dt.subdtype is not None:
+        # h5py expands an HDF5 array element type into trailing array dims
+        dt = dt.subdtype[0]
+    arr = np.asarray(arr, dtype=dt)
+    if simple(dt):
+        c = np.ascontiguousarray(arr).astype(packed(dt)).tobytes()
+    else:
+        out = bytearray()
+        for x in arr.reshape(-1):
+            canon_el(dt, x, out)
+        c = bytes(out)
+    rec["hash"] = hashlib.sha256(c).hexdigest()
+    rec["head"] = c[:48].hex()
+
+
+def err(e):
+    s = f"{type(e).__name__}: {e}"
+    return s.splitlines()[0][:400] if s else type(e).__name__
+
+
+def shape_of(s):
+    return "null" if s is None else list(s)
+
+
+def n_bytes(shape, tid):
+    n = 1
+    for d in shape or ():
+        n *= d
+    return n * tid.get_size()
+
+
+def read_attrs(obj):
+    out = {}
+    names = sorted(obj.attrs.keys(), key=lambda s: s.encode("utf-8", "surrogateescape"))
+    for name in names:
+        rec = {}
+        try:
+            aid = obj.attrs.get_id(name)
+            rec["dtype"] = str(aid.dtype)
+            rec["shape"] = shape_of(aid.shape)
+            note_conversion(aid.get_type(), aid.dtype, rec)
+            if aid.shape is None:
+                hash_values(np.empty((0,), dtype=aid.dtype), aid.dtype, rec)
+            else:
+                val = obj.attrs[name]
+                hash_values(val, aid.dtype, rec)
+        except Exception as e:  # noqa: BLE001
+            rec = {"error": err(e)}
+        out[name] = rec
+    return out
+
+
+def main(path):
+    top = {"file": path}
+    try:
+        f = h5py.File(path, "r")
+    except Exception as e:  # noqa: BLE001
+        top["open_error"] = err(e)
+        print(json.dumps(top))
+        return
+    objects = []
+    seen = set()
+    stack = [("/", None)]
+    while stack:
+        p, obj = stack.pop()
+        if len(objects) >= MAX_OBJECTS:
+            top["truncated"] = True
+            break
+        rec = {"path": p}
+        try:
+            if obj is None:
+                obj = f[p]
+            key = hash(obj.id)  # h5py ObjectID hash = (fileno, object address/token)
+        except Exception as e:  # noqa: BLE001
+            rec["kind"] = "unknown"
+            rec["error"] = err(e)
+            objects.append(rec)
+            continue
+        if key in seen:
+            continue
+        seen.add(key)
+        if isinstance(obj, h5py.Dataset):
+            kind = "dataset"
+        elif isinstance(obj, h5py.Group):
+            kind = "group"
+        elif isinstance(obj, h5py.Datatype):
+            kind = "datatype"
+        else:
+            kind = "unknown"
+        rec["kind"] = kind
+        if kind == "dataset":
+            try:
+                dt = obj.dtype
+                rec["dtype"] = str(dt)
+                rec["shape"] = shape_of(obj.shape)
+                note_conversion(obj.id.get_type(), dt, rec)
+                if obj.shape is None:
+                    hash_values(np.empty((0,), dtype=dt), dt, rec)
+                elif n_bytes(obj.shape, obj.id.get_type()) > MAX_BYTES:
+                    rec["skipped"] = "too large"
+                else:
+                    arr = np.empty(obj.shape, dtype=dt)
+                    if arr.size:
+                        try:
+                            obj.read_direct(arr)
+                        except Exception:  # noqa: BLE001
+                            arr = obj[()]
+                    hash_values(arr, dt, rec)
+            except Exception as e:  # noqa: BLE001
+                rec["error"] = err(e)
+        if kind != "datatype":
+            try:
+                rec["attrs"] = read_attrs(obj)
+            except Exception as e:  # noqa: BLE001
+                rec["attrs_error"] = err(e)
+        if kind == "group":
+            try:
+                names = sorted(obj.keys(), key=lambda s: s.encode("utf-8", "surrogateescape"))
+                base = "" if p == "/" else p
+                kids = []
+                for n in names:
+                    try:
+                        link = obj.get(n, getlink=True)
+                    except Exception:  # noqa: BLE001
+                        link = None
+                    if link is not None and not isinstance(link, h5py.HardLink):
+                        continue
+                    kids.append(f"{base}/{n}")
+                for k in reversed(kids):
+                    stack.append((k, None))
+            except Exception as e:  # noqa: BLE001
+                rec["list_error"] = err(e)
+        objects.append(rec)
+    top["objects"] = objects
+    print(json.dumps(top), flush=True)
+    # Exit without tearing down the h5py objects: freeing them for some files
+    # that hold references (hdf5's h5repack_attr_refs.h5, cve-2024-32623.h5)
+    # makes libhdf5 2.0 abort with "free(): chunks in smallbin corrupted"
+    # about half the time. That happens after the reading is done, so it says
+    # nothing about what h5py read, but it flipped those files between ok and
+    # h5py-cannot-read from one run to the next.
+    os._exit(0)
+
+
+if __name__ == "__main__":
+    main(sys.argv[1])

conformance/report.py

@@ -0,0 +1,335 @@
+#!/usr/bin/env python3
+"""report.py <results_dir> <CONFORMANCE.md> <corpus_dir>
+
+Render the sweep's results (compare.py's results.json plus the raw per-side
+runs) as CONFORMANCE.md, and write <results_dir>/report-meta.json (commit,
+date, versions) for check.py --update.
+"""
+import collections
+import datetime
+import json
+import os
+import platform
+
+import subprocess
+import sys
+
+import h5py
+import numpy
+
+try:
+    import hdf5plugin
+    HDF5PLUGIN = hdf5plugin.version
+except Exception:  # noqa: BLE001
+    HDF5PLUGIN = "not installed"
+
+R, OUT_MD, CORPUS = sys.argv[1], sys.argv[2], sys.argv[3]
+HERE = os.path.dirname(os.path.abspath(__file__))
+ROOT = os.path.dirname(HERE)
+CLASSES = ["ok", "our-error", "mismatch", "h5py-cannot-read", "panic", "hang", "crash", "oom"]
+
+
+def sh(*cmd, cwd=ROOT):
+    try:
+        return subprocess.run(cmd, cwd=cwd, capture_output=True, text=True, timeout=30).stdout.strip()
+    except Exception:  # noqa: BLE001
+        return ""
+
+
+def cpu_model():
+    try:
+        for ln in open("/proc/cpuinfo"):
+            if ln.startswith(("model name", "Model")):
+                return ln.split(":", 1)[1].strip()
+    except OSError:
+        pass
+    return platform.processor() or "unknown"
+
+
+def mem_gib():
+    try:
+        for ln in open("/proc/meminfo"):
+            if ln.startswith("MemTotal:"):
+                return f"{int(ln.split()[1]) / 1048576:.0f} GiB"
+    except OSError:
+        pass
+    return "?"
+
+
+res = json.load(open(os.path.join(R, "results.json")))
+meta_run = json.load(open(os.path.join(R, "meta.json"))) if os.path.exists(os.path.join(R, "meta.json")) else {}
+rows = res["rows"]
+issues = res.get("issues", {})
+
+# safe.directory: a checkout owned by another user (a container) is still ours to read
+commit = sh("git", "-c", "safe.directory=*", "rev-parse", "HEAD") or os.environ.get("GITHUB_SHA", "unknown")
+lib_dirty = sh("git", "-c", "safe.directory=*", "status", "--porcelain", "--", "crates", "Cargo.toml")
+h5dump_v = sh("h5dump", "--version").replace("h5dump: ", "")
+meta = {
+    "date": datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d %H:%M UTC"),
+    "commit": commit + (" (library sources modified)" if lib_dirty else ""),
+    "reference": f"h5py {h5py.__version__} / HDF5 {h5py.version.hdf5_version}",
+}
+json.dump(meta, open(os.path.join(R, "report-meta.json"), "w"), indent=1)
+
+pins = []
+for ln in open(os.path.join(HERE, "corpus.txt")):
+    if ln.strip() and not ln.lstrip().startswith("#"):
+        name, url, rev, root, *_ = ln.split()
+        pins.append((name, url, rev, root))
+
+by_corpus = collections.defaultdict(collections.Counter)
+for r in rows:
+    by_corpus[r["corpus"]][r["class"]] += 1
+total = collections.Counter(r["class"] for r in rows)
+
+
+def ex_list(files, n=3):
+    s = ", ".join(f"`{f}`" for f in files[:n])
+    return s + (f" (+{len(files) - n} more)" if len(files) > n else "")
+
+
+# --- known causes that are not clawhdf5 bugs --------------------------------
+def is_h5py_be_vlen(i):
+    """h5py returns the elements of a VL sequence of a big-endian base type
+    with their file (big-endian) bytes but a native-endian dtype."""
+    return (i["kind"] == "mismatch" and i["key"] in ("values", "attr-values")
+            and (i.get("ref_dtype") == "object") and (i.get("ours_dtype") or "").startswith("vlen(")
+            and ">" in (i.get("ours_dtype") or ""))
+
+
+known = collections.defaultdict(list)
+for r in rows:
+    if r["class"] != "mismatch":
+        continue
+    iss = issues.get(r["file"], [])
+    if iss and all(is_h5py_be_vlen(i) for i in iss):
+        known["h5py-be-vlen"].append(r["file"])
+
+
+# --- the CVE corpus: clawhdf5 vs h5dump vs h5py ------------------------------
+def side(run, name):
+    p = os.path.join(R, "runs", run, name)
+    if not os.path.exists(p + ".rc"):
+        return None
+    rc = int(open(p + ".rc").read().strip() or -1)
+    err = open(p + ".err", errors="replace").read()
+    try:
+        j = json.load(open(p + ".json"))
+    except Exception:  # noqa: BLE001
+        j = None
+    return rc, err, j
+
+
+def outcome(s, rust=False):
+    """-> (bucket, text). bucket in read / error / panic / crash / hang / oom."""
+    if s is None:
+        return "missing", "not run"
+    rc, err, j = s
+    if rc in (137, 124):
+        return "hang", "hang (killed at timeout)"
+    if "memory allocation of" in err or "MemoryError" in err or "bad_alloc" in err or "Cannot allocate" in err:
+        return "oom", "out of memory"
+    if rust and (rc == 101 or "PANIC:" in err):
+        return "panic", "panic"
+    if "overflowed its stack" in err:
+        return "crash", "stack overflow"
+    if rc == 139:
+        return "crash", "SIGSEGV"
+    if rc == 134:
+        return "crash", "SIGABRT" + (" (heap corruption)" if ("corrupted" in err or "free()" in err) else "")
+    if rc > 128:
+        return "crash", f"signal {rc - 128}"
+    if j is None:
+        return ("error", "error exit") if rc in (0, 1) else ("crash", f"exit {rc}")
+    if "open_error" in j:
+        return "error", "open error"
+    objs = j.get("objects", [])
+    ne = sum(1 for o in objs for k in ("error", "attrs_error", "list_error") if k in o)
+    ne += sum(1 for o in objs for a in (o.get("attrs") or {}).values() if "error" in a)
+    return "read", f"read {len(objs)} obj" + (f", {ne} errors" if ne else "")
+
+
+def h5dump_outcome(s):
+    if s is None:
+        return "missing", "not run"
+    rc, err, _ = s
+    if rc in (137, 124):
+        return "hang", "hang (killed at timeout)"
+    if "memory allocation" in err or "Cannot allocate" in err:
+        return "oom", "out of memory"
+    if rc == 139:
+        return "crash", "SIGSEGV"
+    if rc == 134:
+        return "crash", "SIGABRT" + (" (heap corruption)" if ("corrupted" in err or "free()" in err) else "")
+    if rc > 128:
+        return "crash", f"signal {rc - 128}"
+    return ("read", "ok") if rc == 0 else ("error", "error exit")
+
+
+cve_rows = []
+buckets = {"clawhdf5": collections.Counter(), "h5dump": collections.Counter(), "h5py": collections.Counter()}
+ours_panic = {r["file"] for r in rows if r["class"] == "panic"}
+for r in rows:
+    if r["corpus"] != "cve_hdf5":
+        continue
+    run = r["file"].replace("/", "__")
+    o = outcome(side(run, "ours"), rust=True)
+    if o[0] == "read" and r["file"] in ours_panic:
+        o = ("panic", "caught panic")
+    p = outcome(side(run, "ref"))
+    d = h5dump_outcome(side(run, "h5dump"))
+    buckets["clawhdf5"][o[0]] += 1
+    buckets["h5py"][p[0]] += 1
+    buckets["h5dump"][d[0]] += 1
+    cve_rows.append((r["file"].split("/", 1)[1], d[1], p[1], o[1], r["class"]))
+
+# --- render -----------------------------------------------------------------
+L = []
+w = L.append
+w("# clawhdf5 conformance report")
+w("")
+w("Every HDF5 file of eight public corpora (pinned by commit) is read twice — by")
+w("clawhdf5 (`conformance/probe`, the same `clawhdf5-format` calls the facade")
+w("makes) and by h5py/libhdf5 (`conformance/ref.py`) — and the two readings are")
+w("compared object by object: the set of hard-linked objects, each dataset's and")
+w("attribute's shape, and a SHA-256 of its values in a canonical encoding. The")
+w("CVE corpus is also run through `h5dump`. Each side runs under a timeout and an")
+w("address-space limit, so a hang, crash or runaway allocation is recorded, not")
+w("fatal. This file is generated by `conformance/run.sh`; do not edit it by hand.")
+w("")
+w("## Run")
+w("")
+w("| | |")
+w("|---|---|")
+w(f"| date | {meta['date']} |")
+w(f"| clawhdf5 commit | `{meta['commit']}` |")
+w(f"| machine | `{platform.node()}`: {cpu_model()}, {os.cpu_count()} CPUs, {mem_gib()}, {platform.system()} {platform.release()} {platform.machine()} |")
+w(f"| command | `{os.environ.get('CONFORMANCE_CMD', 'conformance/run.sh')}` |")
+w(f"| rustc | {sh('rustc', '-V')} |")
+w(f"| reference | h5py {h5py.__version__}, HDF5 {h5py.version.hdf5_version}, numpy {numpy.__version__}, hdf5plugin {HDF5PLUGIN}, Python {platform.python_version()} |")
+w(f"| h5dump | {h5dump_v} (CVE corpus only) |")
+if meta_run:
+    w(f"| limits | {meta_run.get('timeout_s')} s timeout (SIGKILL), {int(meta_run.get('mem_kb', 0)) // 1024} MiB address space, per process; {meta_run.get('jobs')} files in parallel |")
+    w(f"| runtime | {meta_run.get('probe_seconds')} s probing + comparing ({meta_run.get('build_seconds')} s fetch/build before it) |")
+w("")
+w("## Results")
+w("")
+w("A file's class is the first that applies:")
+w("")
+w("- **panic / hang / crash / oom** — clawhdf5 panicked (caught per object or not), hit the timeout, died on a signal, or failed an allocation. The CI gate fails on any of these.")
+w("- **h5py-cannot-read** — libhdf5 could not open the file (or itself crashed or hung). Nothing to compare against; most are the deliberately malformed CVE reproducers.")
+w("- **our-error** — clawhdf5 returned an error for something h5py reads.")
+w("- **mismatch** — both read it, but the shapes, values, object set or attribute set differ.")
+w("- **ok** — every object h5py reads, clawhdf5 reads identically.")
+w("")
+w("| corpus | files | " + " | ".join(CLASSES) + " |")
+w("|---" * (len(CLASSES) + 2) + "|")
+for c in sorted(by_corpus):
+    cnt = by_corpus[c]
+    w(f"| {c} | {sum(cnt.values())} | " + " | ".join(str(cnt.get(k, 0)) for k in CLASSES) + " |")
+w(f"| **all** | **{len(rows)}** | " + " | ".join(f"**{total.get(k, 0)}**" for k in CLASSES) + " |")
+w("")
+n_known = sum(len(v) for v in known.values())
+if n_known:
+    w(f"{n_known} of the {total.get('mismatch', 0)} mismatches are a known h5py bug, not ours (see *Known not-our-bug*).")
+    w("")
+w("Corpora (fetched by `conformance/fetch-corpus.sh` into the gitignored `conformance/.cache/`):")
+w("")
+w("| corpus | source | commit |")
+w("|---|---|---|")
+for name, url, rev, root in pins:
+    w(f"| {name} | {url.removesuffix('.git')}" + ("" if root == "." else f" (`{root}`)") + f" | `{rev[:12]}` |")
+w("")
+
+w("## Panics, hangs, crashes, out-of-memory")
+w("")
+if not res["panics"]:
+    w("None.")
+else:
+    for p in res["panics"]:
+        w(f"- `{p['file']}` [{p['class']}] {p['detail']}")
+w("")
+
+w("## Our-error root causes")
+w("")
+w("Grouped by normalised error message. *files* counts files whose class this cause affects.")
+w("")
+w("| files | objects | error | examples |")
+w("|---:|---:|---|---|")
+for k, v in res["root_causes"].items():
+    w(f"| {v['files']} | {v['count']} | `{k.replace('|', '/')}` | {ex_list(v['file_list'])} |")
+w("")
+w("## Mismatch root causes")
+w("")
+w("| files | objects | cause | examples |")
+w("|---:|---:|---|---|")
+for k, v in res["mismatch_causes"].items():
+    w(f"| {v['files']} | {v['count']} | `{k.replace('|', '/')}` | {ex_list(v['file_list'])} |")
+w("")
+
+w("## CVE corpus: clawhdf5 vs h5dump vs h5py")
+w("")
+w(f"The {len(cve_rows)} files of [HDFGroup/cve_hdf5](https://github.com/HDFGroup/cve_hdf5) — reproducers for")
+w("published libhdf5 CVEs and fuzzer finds. *read* = produced output (possibly with per-object")
+w("errors), *error* = refused cleanly. h5dump exits non-zero on any error anywhere in a file, so")
+w("its read/error split is not comparable with the other two rows; the panic, crash, hang and oom")
+w("columns are.")
+w("")
+w("| tool | read | error | panic | crash | hang | oom |")
+w("|---|---:|---:|---:|---:|---:|---:|")
+for tool, label in (("clawhdf5", "clawhdf5"), ("h5dump", f"h5dump {h5dump_v.split()[-1] if h5dump_v else ''}"),
+                    ("h5py", f"h5py {h5py.__version__} / HDF5 {h5py.version.hdf5_version}")):
+    b = buckets[tool]
+    w(f"| {label} | " + " | ".join(str(b.get(k, 0)) for k in ("read", "error", "panic", "crash", "hang", "oom")) + " |")
+w("")
+w("<details><summary>Per-file outcomes</summary>")
+w("")
+w("| file | h5dump | h5py | clawhdf5 | class |")
+w("|---|---|---|---|---|")
+for f, d, p, o, cls in cve_rows:
+    w(f"| {f} | {d} | {p} | {o} | {cls} |")
+w("")
+w("</details>")
+w("")
+
+w("## Known not-our-bug")
+w("")
+w("- **h5py big-endian variable-length sequences.** h5py returns the elements of a VL sequence")
+w("  whose base type is big-endian with the file's big-endian bytes but a native (little-endian)")
+w("  numpy dtype, so the values it reports are byte-swapped garbage; `h5dump` prints the values")
+w("  clawhdf5 reads. Reproducer: `h5py.vlen_dtype(np.dtype('>f4'))` dataset holding `[1.0, 2.0]`")
+w("  reads back in h5py as `[4.6e-41, 9.0e-44]`. Affected here: "
+  + (ex_list(sorted(known["h5py-be-vlen"]), 10) if known["h5py-be-vlen"] else "none") + ".")
+w("- **Non-IEEE floats and partial-precision integers (N-Bit).** libhdf5 converts a float whose")
+w("  bit layout is not IEEE (e.g. `H5Tset_precision` for the N-Bit filter) or an integer with a")
+w("  bit offset / reduced precision into the plain numpy type of the same size. The probe")
+w("  compares such values as converted numbers, not raw file bytes (before 2026-09-25 it compared")
+w("  raw bytes, which reported every N-Bit float dataset as a mismatch).")
+if res["incomparable"]:
+    w("- **Types h5py widens.** Where h5py reads a type into a numpy type of a different size")
+    w("  (FP8 -> float16, bfloat16 -> float32, x87 long double -> float128) the values are not")
+    w("  compared (shape and presence still are): "
+      + ", ".join(f"{k} ({n}x)" for k, n in res["incomparable"]) + ".")
+w("- **References** are compared by presence only (`R`), not by target.")
+w("")
+if res.get("ref_only_errors"):
+    w("## Objects h5py fails on but clawhdf5 reads")
+    w("")
+    for k, n in res["ref_only_errors"][:15]:
+        w(f"- {n} x `{k}`")
+    w("")
+w("## Reproduce")
+w("")
+w("```sh")
+w("# needs: Rust, python3 with h5py numpy hdf5plugin (conformance/requirements.txt), h5dump (hdf5-tools), git")
+w("CLAWHDF5_PYTHON=/path/to/venv/bin/python conformance/run.sh")
+w("```")
+w("")
+w("The corpus (about 450 MB of sparse checkouts) is cached in `conformance/.cache/`; results for")
+w("every file, both sides' raw JSON and stderr, are in `conformance/.cache/results/`.")
+w("`conformance/baseline.json` holds the ok files the nightly CI job (`.gitea/workflows/conformance.yml`)")
+w("must keep; `conformance/run.sh --update-baseline` rewrites it.")
+
+with open(OUT_MD, "w") as fh:
+    fh.write("\n".join(L) + "\n")

conformance/requirements.txt

@@ -0,0 +1,6 @@
+# The reference side of the conformance sweep. Pinned so the nightly job and a
+# local run compare against the same libhdf5 (h5py wheels bundle it).
+h5py==3.16.0
+numpy==2.5.3
+hdf5plugin==7.1.0
+netCDF4==1.7.4

conformance/run.sh

@@ -0,0 +1,88 @@
+#!/usr/bin/env bash
+# conformance/run.sh — the clawhdf5 conformance sweep, end to end.
+#
+#   fetch the pinned corpora (cached) -> build the probe -> probe every file
+#   with clawhdf5 and with h5py (and h5dump for the CVE corpus), each under a
+#   timeout and a memory limit -> compare -> write CONFORMANCE.md -> check the
+#   result against conformance/baseline.json.
+#
+# Usage: conformance/run.sh [--no-fetch] [--no-report] [--update-baseline]
+#
+# Environment:
+#   CLAWHDF5_PYTHON    python with h5py, numpy, hdf5plugin (default: repo .venv, then python3)
+#   CONFORMANCE_CACHE  corpus / build / results cache (default: conformance/.cache)
+#   CONFORMANCE_OUT    results directory (default: $CONFORMANCE_CACHE/results)
+#   CONFORMANCE_REPORT report path (default: CONFORMANCE.md at the repo root)
+#   JOBS               parallel files (default: nproc)
+#   CONFORMANCE_PROBE  use this prebuilt probe binary instead of building one
+#   TMO / MEM_KB       per-process timeout in seconds (20) / address-space limit in KiB (4 GiB)
+#
+# Exit status: 0 = gate passed; 1 = a panic/hang/crash/oom in clawhdf5, or the
+# ok count fell below the baseline, or a baseline-ok file regressed; 2 = setup error.
+set -euo pipefail
+HERE="$(cd "$(dirname "$0")" && pwd)"
+ROOT="$(cd "$HERE/.." && pwd)"
+FETCH=1 REPORT=1 UPDATE=0
+for a in "$@"; do
+  case "$a" in
+    --no-fetch) FETCH=0 ;;
+    --no-report) REPORT=0 ;;
+    --update-baseline) UPDATE=1 ;;
+    -h|--help) sed -n '2,23p' "$0"; exit 0 ;;
+    *) echo "unknown argument: $a" >&2; exit 2 ;;
+  esac
+done
+
+export PATH="$HOME/.cargo/bin:$PATH"
+CACHE="${CONFORMANCE_CACHE:-$HERE/.cache}"
+mkdir -p "$CACHE"; CACHE="$(cd "$CACHE" && pwd)"
+OUT="${CONFORMANCE_OUT:-$CACHE/results}"
+REPORT_PATH="${CONFORMANCE_REPORT:-$ROOT/CONFORMANCE.md}"
+JOBS="${JOBS:-$(nproc 2>/dev/null || echo 4)}"
+if [ -n "${CLAWHDF5_PYTHON:-}" ]; then PY="$CLAWHDF5_PYTHON"
+elif [ -x "$ROOT/.venv/bin/python" ]; then PY="$ROOT/.venv/bin/python"
+else PY="$(command -v python3)"; fi
+export PY TMO="${TMO:-20}" MEM_KB="${MEM_KB:-4194304}"
+command -v h5dump >/dev/null || { echo "error: h5dump not found (install hdf5-tools)" >&2; exit 2; }
+"$PY" -c 'import h5py, numpy, hdf5plugin' || { echo "error: $PY lacks h5py/numpy/hdf5plugin" >&2; exit 2; }
+
+t0=$(date +%s)
+[ "$FETCH" = 1 ] && bash "$HERE/fetch-corpus.sh" "$CACHE"
+C="$CACHE/corpus"
+[ -d "$C" ] || { echo "error: no corpus in $C (run without --no-fetch)" >&2; exit 2; }
+
+if [ -n "${CONFORMANCE_PROBE:-}" ]; then
+  export PROBE="$CONFORMANCE_PROBE"   # a prebuilt probe, e.g. an older one for a before/after
+else
+  echo "== building the probe"
+  CARGO_TARGET_DIR="${CARGO_TARGET_DIR:-$CACHE/target}" \
+    cargo build -q --release --manifest-path "$HERE/probe/Cargo.toml"
+  export PROBE="${CARGO_TARGET_DIR:-$CACHE/target}/release/conformance-probe"
+fi
+t1=$(date +%s)
+
+rm -rf "$OUT"; mkdir -p "$OUT"
+"$PY" "$HERE/list_files.py" "$C" > "$OUT/files.txt"
+echo "== probing $(wc -l <"$OUT/files.txt") files, $JOBS at a time (timeout ${TMO}s, limit $((MEM_KB / 1024)) MiB)"
+export C OUT HERE
+# The shell's "Segmentation fault (core dumped)" notices go to probe.log; the
+# signals themselves are recorded in each side's .rc.
+xargs -a "$OUT/files.txt" -d '\n' -P "$JOBS" -I{} bash -c '
+  f="$1"; d="$OUT/runs/${f//\//__}"
+  case "$f" in cve_hdf5/*) export WITH_H5DUMP=1 ;; esac
+  "$HERE/run_one.sh" "$C/$f" "$d"' _ {} 2>"$OUT/probe.log"
+echo "== comparing"
+"$PY" "$HERE/compare.py" "$OUT" >/dev/null
+t2=$(date +%s)
+cat > "$OUT/meta.json" <<EOF
+{"build_seconds": $((t1 - t0)), "probe_seconds": $((t2 - t1)), "jobs": $JOBS, "timeout_s": $TMO, "mem_kb": $MEM_KB}
+EOF
+export CONFORMANCE_CMD="${CONFORMANCE_CMD:-conformance/run.sh${*:+ $*}}"
+if [ "$REPORT" = 1 ]; then
+  "$PY" "$HERE/report.py" "$OUT" "$REPORT_PATH" "$C"
+  echo "== wrote $REPORT_PATH"
+fi
+if [ "$UPDATE" = 1 ]; then
+  "$PY" "$HERE/check.py" "$OUT" "$HERE/baseline.json" --update
+fi
+"$PY" "$HERE/check.py" "$OUT" "$HERE/baseline.json"

conformance/run_one.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+# run_one.sh <file> <outdir>
+#
+# Probe one file with clawhdf5 (PROBE) and with h5py (PY ref.py), and with
+# h5dump too when WITH_H5DUMP is set. Each side runs under a timeout (TMO
+# seconds, SIGKILL) and an address-space limit (MEM_KB), with core dumps off.
+# Writes <outdir>/<side>.{json,err,rc}; rc 137 = killed by the timeout.
+set -u
+f="$1"; out="$2"; mkdir -p "$out"
+HERE="$(cd "$(dirname "$0")" && pwd)"
+: "${PROBE:?PROBE must name the conformance-probe binary}"
+: "${PY:?PY must name a python with h5py}"
+TMO="${TMO:-20}"
+MEM_KB="${MEM_KB:-4194304}"
+run() { # name cmd...
+  local name=$1; shift
+  ( ulimit -v "$MEM_KB"; ulimit -c 0; RUST_BACKTRACE=1 exec timeout -s KILL "$TMO" "$@" ) \
+      >"$out/$name.json" 2>"$out/$name.err"
+  echo $? >"$out/$name.rc"
+}
+run ours "$PROBE" "$f"
+run ref "$PY" "$HERE/ref.py" "$f"
+if [ -n "${WITH_H5DUMP:-}" ]; then
+  run h5dump h5dump "$f"
+  : >"$out/h5dump.json" # h5dump's text dump is not compared, only its exit status
+fi
+exit 0

crates/clawhdf5-agent/src/schema.rs

@@ -477,10 +477,34 @@ fn write_string_dataset(
     }
 }
 
+/// `/meta`'s attributes, failing if any of them cannot be read.
+///
+/// `Group::attrs` leaves out an attribute it cannot decode. For the store's
+/// settings that would silently fall back to defaults (e.g. `float16`, the
+/// WAL mark), so an unreadable attribute is an error here, as it was before
+/// `attrs` became tolerant.
+fn meta_attrs(
+    file: &clawhdf5::File,
+) -> Result<std::collections::HashMap<String, AttrValue>, MemoryError> {
+    let meta = file
+        .group("meta")
+        .map_err(|e| MemoryError::Schema(format!("missing /meta group: {e}")))?;
+    let (attrs, errors) = meta
+        .attrs_with_errors()
+        .map_err(|e| MemoryError::Schema(format!("cannot read /meta attrs: {e}")))?;
+    if let Some(e) = errors.first() {
+        return Err(MemoryError::Schema(format!(
+            "cannot read /meta attrs: {} unreadable, first: {e}",
+            errors.len()
+        )));
+    }
+    Ok(attrs)
+}
+
 /// Validate an HDF5 file has the correct schema and load all data.
 /// Read the checkpoint's [`WalMark`] from `/meta`, if it has one.
 pub fn read_wal_mark(file: &clawhdf5::File) -> Option<WalMark> {
-    let attrs = file.group("meta").ok()?.attrs().ok()?;
+    let attrs = meta_attrs(file).ok()?;
     let len = match attrs.get(WAL_APPLIED_LEN_ATTR)? {
         AttrValue::I64(v) => u64::try_from(*v).ok()?,
         _ => return None,
@@ -498,10 +522,7 @@ pub fn read_signature(
     file: &clawhdf5::File,
 ) -> Result<Option<crate::signing::StoredSignature>, MemoryError> {
     use crate::signing::{Manifest, StoredSignature, from_hex};
-    let attrs = file
+    let attrs = meta_attrs(file)?;
-        .group("meta")
-        .and_then(|g| g.attrs())
-        .map_err(|e| MemoryError::Schema(format!("cannot read /meta attrs: {e}")))?;
     let version = match attrs.get(SIG_VERSION_ATTR) {
         None => return Ok(None),
         Some(AttrValue::I64(v)) => *v,
@@ -552,18 +573,14 @@ pub fn read_signature(
 
 /// Read the checkpoint bookkeeping from `/meta`.
 pub fn read_checkpoint_meta(file: &clawhdf5::File) -> CheckpointMeta {
-    let ann_generation = file
+    let ann_generation =
-        .group("meta")
+        meta_attrs(file)
             .ok()
-        .and_then(|g| g.attrs().ok())
             .and_then(|attrs| match attrs.get(ANN_GENERATION_ATTR) {
                 Some(AttrValue::I64(v)) => Some(*v as u64),
                 _ => None,
             });
-    let signed = file
+    let signed = meta_attrs(file).is_ok_and(|attrs| attrs.contains_key(SIG_VERSION_ATTR));
-        .group("meta")
-        .and_then(|g| g.attrs())
-        .is_ok_and(|attrs| attrs.contains_key(SIG_VERSION_ATTR));
     CheckpointMeta {
         wal_applied: read_wal_mark(file),
         ann_generation,
@@ -575,12 +592,7 @@ pub fn validate_and_load(
     file: &clawhdf5::File,
 ) -> Result<(MemoryConfig, MemoryCache, SessionCache, KnowledgeCache), MemoryError> {
     // Read /meta group attributes
-    let meta = file
+    let attrs = meta_attrs(file)?;
-        .group("meta")
-        .map_err(|e| MemoryError::Schema(format!("missing /meta group: {e}")))?;
-    let attrs = meta
-        .attrs()
-        .map_err(|e| MemoryError::Schema(format!("cannot read /meta attrs: {e}")))?;
 
     let schema_version = match attrs.get("schema_version") {
         Some(AttrValue::String(s)) => s.clone(),

crates/clawhdf5-agent/tests/float16_store.rs

@@ -258,3 +258,43 @@ fn an_existing_f32_store_stays_f32() {
     assert_eq!(&values[..before.1.len()], before.1.as_slice());
     assert_eq!(&values[before.1.len()..], odd.as_slice());
 }
+
+/// `Group::attrs` leaves out an attribute it cannot decode. A store whose
+/// `float16` setting is unreadable must not open as `float16 = false` (or with
+/// any other default in place of a setting it has): it is an error.
+#[test]
+fn unreadable_meta_attribute_fails_open_instead_of_defaulting() {
+    let dir = TempDir::new().unwrap();
+    let path = dir.path().join("store.h5");
+    {
+        let mut m = HDF5Memory::create(config(&dir, "store.h5", true)).unwrap();
+        m.save(entry(1)).unwrap();
+        m.flush_wal().unwrap();
+    }
+    assert!(HDF5Memory::open_read_only(&path).is_ok());
+
+    // Give the `float16` attribute message an unknown version (the name is
+    // at +8 in a version-1 message and +9 in a version-3 one).
+    let mut bytes = std::fs::read(&path).unwrap();
+    let name = b"float16\0";
+    let mut hit = false;
+    let positions: Vec<usize> = (9..bytes.len() - name.len())
+        .filter(|&p| &bytes[p..p + name.len()] == name)
+        .collect();
+    for pos in positions {
+        for (back, version) in [(8, 1u8), (9, 3u8)] {
+            if bytes[pos - back] == version {
+                bytes[pos - back] = 0x7f;
+                hit = true;
+            }
+        }
+    }
+    assert!(hit, "float16 attribute message not found");
+    std::fs::write(&path, &bytes).unwrap();
+
+    match HDF5Memory::open_read_only(&path) {
+        Err(MemoryError::Schema(msg)) => assert!(msg.contains("/meta"), "{msg}"),
+        Err(e) => panic!("unexpected error: {e}"),
+        Ok(_) => panic!("store opened with an unreadable float16 setting"),
+    }
+}

crates/clawhdf5-ann/src/hnsw.rs

@@ -13,7 +13,7 @@ use clawhdf5_format::filter_pipeline::FilterPipeline;
 use clawhdf5_format::group_v2::resolve_path_any;
 use clawhdf5_format::message_type::MessageType;
 use clawhdf5_format::object_header::ObjectHeader;
-use clawhdf5_format::signature::find_signature;
+use clawhdf5_format::signature::split_user_block;
 use clawhdf5_format::superblock::Superblock;
 use clawhdf5_io::FileWriter as IoFileWriter;
 
@@ -861,8 +861,9 @@ impl HnswIndex {
     /// The HDF5 data must contain the `/ann/vectors`, `/ann/graph_layer_*`,
     /// and `/ann/config` datasets as produced by [`to_hdf5_bytes`].
     pub fn load_from_hdf5(data: &[u8]) -> Result<Self, FormatError> {
-        let sig_offset = find_signature(data)?;
+        // Addresses are relative to the superblock: skip any user block.
-        let sb = Superblock::parse(data, sig_offset)?;
+        let (_, data) = split_user_block(data)?;
+        let sb = Superblock::parse(data, 0)?;
 
         // Read config dataset and its attributes
         let config_attrs = read_dataset_attrs(data, &sb, "ann/config")?;

crates/clawhdf5-bench/Cargo.toml

@@ -34,6 +34,10 @@ path = "src/bin/consolidation_efficiency.rs"
 name = "ephemeral_perf"
 path = "src/bin/ephemeral_perf.rs"
 
+[[bin]]
+name = "concurrent_read"
+path = "src/bin/concurrent_read.rs"
+
 [[bin]]
 name = "mpi_io_bench"
 path = "src/bin/mpi_io_bench.rs"
@@ -64,6 +68,10 @@ clawhdf5-io = { path = "../clawhdf5-io" }
 mpi = { version = "0.8", optional = true }
 serde = { workspace = true }
 serde_json = "1"
+# concurrent_read: size the decode pool (--decode-threads) and evict files
+# from the page cache (--cold, posix_fadvise). Both pure Rust / bindings only.
+rayon = "1"
+libc = "0.2"
 tempfile = { workspace = true }
 # Optional: libhdf5 C wrapper for side-by-side comparison (requires system libhdf5).
 # Enable with: cargo bench -p clawhdf5-bench --features libhdf5-compare

crates/clawhdf5-bench/scripts/compare_concurrent_read.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+"""Tabulate concurrent_read JSON results (clawhdf5, h5py threads/processes).
+
+    python compare_concurrent_read.py clawhdf5.json h5py-threads.json h5py-procs.json
+
+Prints one Markdown table: for each layout, mode and thread count, every
+tool's MB/s and scaling efficiency, and the first file's MB/s relative to each
+of the others. Refuses to compare runs whose workload parameters differ.
+"""
+
+import json
+import sys
+
+COMPARED = ("datasets", "rows", "cols", "chunk", "deflate_level", "slab", "slabs", "seed")
+
+
+def main(paths):
+    if len(paths) < 2:
+        sys.exit(__doc__)
+    docs = []
+    for p in paths:
+        with open(p) as fh:
+            docs.append(json.load(fh))
+    ref = docs[0]
+    for d, p in zip(docs[1:], paths[1:]):
+        diff = [k for k in COMPARED if d["params"].get(k) != ref["params"].get(k)]
+        if diff:
+            sys.exit(f"{p}: workload differs from {paths[0]} in {', '.join(diff)}")
+        if d["cache"] != ref["cache"]:
+            print(f"warning: {p} ran {d['cache']!r}, {paths[0]} ran {ref['cache']!r}",
+                  file=sys.stderr)
+        if d.get("host") != ref.get("host"):
+            print(f"warning: {p} ran on {d.get('host')}, {paths[0]} on {ref.get('host')}",
+                  file=sys.stderr)
+
+    names = [d["tool"] for d in docs]
+    for d in docs:
+        extra = f", HDF5 {d['hdf5_version']}" if "hdf5_version" in d else ""
+        print(f"- {d['tool']} {d['version']}{extra}: host {d.get('host')}, "
+              f"{d.get('cpus')} CPUs, cache {d['cache']}, decode threads per read "
+              f"{d.get('decode_threads')}")
+    p = ref["params"]
+    print(f"\n{p['datasets']} datasets of {p['rows']} x {p['cols']} f32, chunks "
+          f"{p['chunk'][0]} x {p['chunk'][1]} (deflate {p['deflate_level']}); "
+          f"`same`: {p['slabs']} slabs of {p['slab']} x {p['slab']}\n")
+
+    index = [{(r["layout"], r["mode"], r["threads"]): r for r in d["results"]} for d in docs]
+    keys = [(r["layout"], r["mode"], r["threads"]) for r in ref["results"]]
+
+    head = ["layout", "mode", "threads"]
+    head += [f"{n} MB/s (eff)" for n in names]
+    head += [f"{names[0]} / {n}" for n in names[1:]]
+    print("| " + " | ".join(head) + " |")
+    print("|---|---|" + "---:|" * (len(head) - 2))
+    for key in keys:
+        cells = [key[0], key[1], str(key[2])]
+        rs = [ix.get(key) for ix in index]
+        for r in rs:
+            if r is None:
+                cells.append("-")
+            else:
+                eff = "-" if r["efficiency"] is None else f"{r['efficiency']:.2f}"
+                cells.append(f"{r['mb_s']:.0f} ({eff})")
+        for r in rs[1:]:
+            cells.append("-" if r is None else f"{rs[0]['mb_s'] / r['mb_s']:.2f}x")
+        print("| " + " | ".join(cells) + " |")
+
+
+if __name__ == "__main__":
+    main(sys.argv[1:])

crates/clawhdf5-bench/scripts/concurrent_read_h5py.py

@@ -0,0 +1,265 @@
+#!/usr/bin/env python3
+"""The concurrent_read workload with h5py, on the files concurrent_read wrote.
+
+libhdf5 serialises every API call under one global lock, and h5py holds its
+own global lock around every call as well, so h5py *threads* cannot decode in
+parallel. h5py users scale with *processes* instead; ``--executor processes``
+measures that (each worker opens the file itself).
+
+The workload mirrors ``crates/clawhdf5-bench/src/bin/concurrent_read.rs``:
+
+* ``distinct``: every dataset read in full once per repetition; worker ``t``
+  of ``T`` reads datasets ``t, t + T, ...``.
+* ``same``: ``--slabs`` random ``--slab`` x ``--slab`` hyperslabs of ``d00``
+  (slab ``j`` to worker ``j % T``), offsets from the same splitmix64 stream.
+
+Each worker times itself from a start barrier; a repetition spans the earliest
+start to the latest finish (CLOCK_MONOTONIC, comparable across processes).
+Threads share one ``h5py.File`` per repetition; process workers open the file
+inside the timed region (a few ms against reads of many MiB).
+
+Generate the files first with the Rust harness (it writes ``manifest.json``),
+then, for example::
+
+    python concurrent_read_h5py.py --dir DIR --executor threads --json h5py-threads.json
+    python concurrent_read_h5py.py --dir DIR --executor processes --json h5py-procs.json
+"""
+
+import argparse
+import json
+import multiprocessing as mp
+import os
+import platform
+import socket
+import sys
+import threading
+import time
+
+import h5py
+import numpy as np
+
+M64 = (1 << 64) - 1
+
+
+def splitmix64(state):
+    """Return (new_state, value); the same stream as the Rust harness."""
+    state = (state + 0x9E3779B97F4A7C15) & M64
+    z = state
+    z = ((z ^ (z >> 30)) * 0xBF58476D1CE4E5B9) & M64
+    z = ((z ^ (z >> 27)) * 0x94D049BB133111EB) & M64
+    return state, z ^ (z >> 31)
+
+
+def value(k, i):
+    """Element i (row-major) of dataset k, exactly as concurrent_read writes it."""
+    _, noise = splitmix64(i ^ (k << 40))
+    return np.float32((((i >> 6) % 16384) + k) + (noise & 0xFF) / 256.0)
+
+
+def slab_offsets(seed, count, rows, cols, slab):
+    s = seed
+    out = []
+    for _ in range(count):
+        s, r = splitmix64(s)
+        s, c = splitmix64(s)
+        out.append((r % (rows - slab + 1), c % (cols - slab + 1)))
+    return out
+
+
+def now():
+    return time.clock_gettime(time.CLOCK_MONOTONIC)
+
+
+def work(f, mode, t, threads, m, slabs, slab, verify):
+    """Worker t's share of one repetition on an open h5py.File."""
+    n = m["rows"] * m["cols"]
+    if mode == "distinct":
+        for k in range(t, m["datasets"], threads):
+            got = f[f"d{k:02d}"][...]
+            assert got.size == n
+            if verify:
+                flat = got.reshape(-1)
+                for i in (0, n // 3, n - 1):
+                    assert flat[i] == value(k, i), f"d{k:02d}[{i}]"
+    else:
+        ds = f["d00"]
+        cols = m["cols"]
+        for r, c in slabs[t::threads]:
+            got = ds[r : r + slab, c : c + slab]
+            assert got.shape == (slab, slab)
+            if verify:
+                assert got[0, 0] == value(0, r * cols + c)
+                last = (r + slab - 1) * cols + c + slab - 1
+                assert got[-1, -1] == value(0, last)
+
+
+# ----- process workers ------------------------------------------------------
+
+_barrier = None
+
+
+def _init(barrier):
+    global _barrier
+    _barrier = barrier
+
+
+def _proc_task(task):
+    path, mode, t, threads, m, slabs, slab = task
+    _barrier.wait()
+    start = now()
+    with h5py.File(path, "r") as f:
+        work(f, mode, t, threads, m, slabs, slab, False)
+    return start, now()
+
+
+def _noop(_):
+    return os.getpid()
+
+
+def run_threads(path, mode, threads, m, slabs, slab):
+    spans = [None] * threads
+    barrier = threading.Barrier(threads)
+    with h5py.File(path, "r") as f:
+
+        def body(t):
+            barrier.wait()
+            start = now()
+            work(f, mode, t, threads, m, slabs, slab, False)
+            spans[t] = (start, now())
+
+        ts = [threading.Thread(target=body, args=(t,)) for t in range(threads)]
+        for th in ts:
+            th.start()
+        for th in ts:
+            th.join()
+    return max(e for _, e in spans) - min(s for s, _ in spans)
+
+
+def run_processes(pool, path, mode, threads, m, slabs, slab):
+    tasks = [(path, mode, t, threads, m, slabs, slab) for t in range(threads)]
+    # One task per worker: each blocks in the barrier until all T have
+    # started, so no worker can take a second task.
+    spans = pool.map(_proc_task, tasks, chunksize=1)
+    return max(e for _, e in spans) - min(s for s, _ in spans)
+
+
+def warm(path):
+    with open(path, "rb") as fh:
+        while fh.read(1 << 24):
+            pass
+
+
+def evict(path):
+    fd = os.open(path, os.O_RDONLY)
+    try:
+        os.posix_fadvise(fd, 0, 0, os.POSIX_FADV_DONTNEED)
+    finally:
+        os.close(fd)
+
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
+    ap.add_argument("--dir", default="concurrent-read-data")
+    ap.add_argument("--executor", choices=["threads", "processes"], default="threads")
+    ap.add_argument("--threads", default="1,2,4,8,16")
+    ap.add_argument("--reps", type=int, default=3)
+    ap.add_argument("--slab", type=int, default=256)
+    ap.add_argument("--slabs", type=int, default=1024)
+    ap.add_argument("--seed", type=int, default=42)
+    ap.add_argument("--cold", action="store_true")
+    ap.add_argument("--modes", default="distinct,same")
+    ap.add_argument("--layouts", default="deflate,contiguous")
+    ap.add_argument("--json")
+    a = ap.parse_args()
+
+    # The Rust harness pins this value (splitmix64_reference).
+    assert splitmix64(42)[1] == 0xBDD732262FEB6E95, "splitmix64 port is wrong"
+
+    try:
+        with open(os.path.join(a.dir, "manifest.json")) as fh:
+            m = json.load(fh)
+    except FileNotFoundError:
+        sys.exit(f"{a.dir}/manifest.json not found: generate the files with "
+                 "`cargo run --release -p clawhdf5-bench --bin concurrent_read -- --dir ...` first")
+    threads_list = [int(x) for x in a.threads.split(",")]
+    modes = a.modes.split(",")
+    layouts = a.layouts.split(",")
+    if a.slab < 1 or a.slab > min(m["rows"], m["cols"]):
+        sys.exit(f"--slab must be 1..={min(m['rows'], m['cols'])}")
+    files = dict(m["files"])
+    slabs = slab_offsets(a.seed, a.slabs, m["rows"], m["cols"], a.slab)
+    dataset_bytes = m["rows"] * m["cols"] * 4
+    tool = f"h5py-{a.executor}"
+
+    ctx = mp.get_context("spawn")  # never fork a process holding HDF5 state
+    pools = {}
+    if a.executor == "processes":
+        for t in threads_list:
+            pool = ctx.Pool(t, initializer=_init, initargs=(ctx.Barrier(t),))
+            pool.map(_noop, range(t))  # start the workers outside the timing
+            pools[t] = pool
+
+    rows = []
+    print("| layout | mode | threads | MB/s | efficiency | median s |")
+    print("|---|---|---:|---:|---:|---:|")
+    try:
+        for layout in layouts:
+            path = os.path.join(a.dir, files[layout])
+            if not a.cold:
+                warm(path)
+            for mode in modes:
+                with h5py.File(path, "r") as f:  # untimed, checked pass
+                    work(f, mode, 0, 1, m, slabs, a.slab, True)
+                nbytes = (dataset_bytes * m["datasets"] if mode == "distinct"
+                          else a.slab * a.slab * 4 * a.slabs)
+                base = None
+                for t in threads_list:
+                    times = []
+                    for _ in range(a.reps):
+                        if a.cold:
+                            evict(path)
+                        if a.executor == "threads":
+                            times.append(run_threads(path, mode, t, m, slabs, a.slab))
+                        else:
+                            times.append(run_processes(pools[t], path, mode, t, m, slabs, a.slab))
+                    med = sorted(times)[len(times) // 2]
+                    mb_s = nbytes / (1 << 20) / med
+                    if t == 1:
+                        base = mb_s
+                    eff = mb_s / (t * base) if base else None
+                    print(f"| {layout} | {mode} | {t} | {mb_s:.0f} | "
+                          f"{'-' if eff is None else f'{eff:.2f}'} | {med:.4f} |")
+                    rows.append({
+                        "layout": layout, "mode": mode, "threads": t, "bytes": nbytes,
+                        "times_s": times, "median_s": med, "mb_s": mb_s, "efficiency": eff,
+                    })
+    finally:
+        for pool in pools.values():
+            pool.terminate()
+
+    if a.json:
+        doc = {
+            "tool": tool,
+            "version": h5py.__version__,
+            "hdf5_version": h5py.version.hdf5_version,
+            "python": platform.python_version(),
+            "host": socket.gethostname(),
+            "cpus": os.cpu_count(),
+            "unix_time": int(time.time()),
+            "cache": ("cold (posix_fadvise DONTNEED before each repetition)"
+                      if a.cold else "warm"),
+            "decode_threads": 1,
+            "params": {
+                "datasets": m["datasets"], "rows": m["rows"], "cols": m["cols"],
+                "chunk": m["chunk"], "deflate_level": m["deflate_level"],
+                "mib": dataset_bytes // (1 << 20), "slab": a.slab, "slabs": a.slabs,
+                "seed": a.seed, "reps": a.reps, "dir": a.dir,
+            },
+            "results": rows,
+        }
+        with open(a.json, "w") as fh:
+            json.dump(doc, fh, indent=2)
+
+
+if __name__ == "__main__":
+    main()

crates/clawhdf5-bench/src/bin/concurrent_read.rs

@@ -0,0 +1,523 @@
+//! Concurrent-read harness: how does decoded read throughput scale with the
+//! number of threads reading one open file?
+//!
+//! libhdf5 (threadsafe build) serialises every API call under one global
+//! mutex, and h5py holds it too, so threads cannot decode in parallel there.
+//! A clawhdf5 [`File`] is `Send + Sync`; this harness measures what that buys.
+//! `crates/clawhdf5-bench/scripts/concurrent_read_h5py.py` runs the same
+//! workload on the same files with h5py (threads, and processes), and
+//! `compare_concurrent_read.py` tabulates the JSON both write.
+//!
+//! Files (generated on first use, reused while `manifest.json` matches):
+//!
+//! * `<dir>/deflate.h5`: `--datasets` datasets `d00`, `d01`, ... of `f32`,
+//!   `--mib` MiB decoded each, shape `[mib * 256, 1024]`, chunks `256 x 256`,
+//!   deflate level 4.
+//! * `<dir>/contiguous.h5`: the same datasets, contiguous.
+//!
+//! Modes, for each layout and each thread count `T` (strong scaling: the total
+//! work per repetition is fixed, split among the threads):
+//!
+//! * `distinct`: every dataset is read in full once; thread `t` reads datasets
+//!   `t, t + T, t + 2T, ...`.
+//! * `same`: all threads read `d00`, `--slabs` random `--slab` x `--slab`
+//!   hyperslabs in total (slab `j` goes to thread `j % T`). The offsets come
+//!   from a splitmix64 stream seeded with `--seed`, identical in the h5py
+//!   script.
+//!
+//! One `File` per layout per repetition is shared by all threads (opened
+//! fresh each repetition, so no chunk cache carries over). Page cache:
+//! `warm` (default) reads every file once before timing; `--cold` evicts the
+//! files from the page cache with `posix_fadvise(POSIX_FADV_DONTNEED)` before
+//! every repetition (no root needed; it only evicts clean, unmapped pages, so
+//! it is best effort — the JSON says which was used).
+//!
+//! Decode inside one read is itself parallel when clawhdf5-format's `parallel`
+//! feature is on (it is in this binary, via clawhdf5-agent). `--decode-threads
+//! N` sizes that rayon pool; `--decode-threads 1` measures the API's own
+//! thread scaling, comparable with h5py where each call decodes on the
+//! calling thread.
+//!
+//! ```text
+//! cargo run --release -p clawhdf5-bench --bin concurrent_read -- \
+//!     --dir /data/concurrent-read --json clawhdf5.json
+//! cargo run --release -p clawhdf5-bench --bin concurrent_read -- \
+//!     --dir /tmp/cr --datasets 4 --mib 1 --threads 1,2 --slabs 16 --reps 1   # smoke
+//! ```
+
+use std::path::{Path, PathBuf};
+use std::sync::Barrier;
+use std::time::Instant;
+
+use clawhdf5::{File, FileBuilder, Selection};
+use serde::{Deserialize, Serialize};
+
+const COLS: u64 = 1024;
+const ROWS_PER_MIB: u64 = 256; // 256 rows x 1024 cols x 4 bytes = 1 MiB
+const CHUNK: u64 = 256;
+const DEFLATE_LEVEL: u32 = 4;
+const LAYOUTS: [&str; 2] = ["deflate", "contiguous"];
+const MANIFEST_VERSION: u32 = 1;
+
+/// splitmix64 — shared with the h5py script, which must produce the same
+/// stream (both the data and the hyperslab offsets depend on it).
+fn splitmix64(state: &mut u64) -> u64 {
+    *state = state.wrapping_add(0x9E37_79B9_7F4A_7C15);
+    let mut z = *state;
+    z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
+    z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
+    z ^ (z >> 31)
+}
+
+/// Element `i` (row-major) of dataset `k`: a slowly varying integer part plus
+/// 8 bits of noise, so deflate has real work to do (about 3.1x) and every value
+/// is exact in `f32` (< 2^15 with 8 fraction bits), which lets both harnesses
+/// check what they read against this formula.
+fn value(k: u64, i: u64) -> f32 {
+    let mut s = i ^ (k << 40);
+    let noise = splitmix64(&mut s) & 0xff;
+    (((i >> 6) % 16384) + k) as f32 + noise as f32 / 256.0
+}
+
+#[derive(Serialize, Deserialize, PartialEq, Debug, Clone)]
+struct Manifest {
+    version: u32,
+    datasets: u64,
+    rows: u64,
+    cols: u64,
+    chunk: [u64; 2],
+    deflate_level: u32,
+    files: Vec<(String, String)>, // (layout, file name)
+    writer: String,
+}
+
+fn manifest_for(datasets: u64, mib: u64) -> Manifest {
+    Manifest {
+        version: MANIFEST_VERSION,
+        datasets,
+        rows: mib * ROWS_PER_MIB,
+        cols: COLS,
+        chunk: [CHUNK, CHUNK],
+        deflate_level: DEFLATE_LEVEL,
+        files: LAYOUTS
+            .iter()
+            .map(|l| (l.to_string(), format!("{l}.h5")))
+            .collect(),
+        writer: format!("clawhdf5 {}", env!("CARGO_PKG_VERSION")),
+    }
+}
+
+fn dataset_values(k: u64, n: u64) -> Vec<f32> {
+    (0..n).map(|i| value(k, i)).collect()
+}
+
+/// Write the files unless `dir` already holds ones matching `want`.
+fn ensure_files(dir: &Path, want: &Manifest) -> std::io::Result<bool> {
+    let manifest_path = dir.join("manifest.json");
+    if let Ok(text) = std::fs::read_to_string(&manifest_path)
+        && let Ok(have) = serde_json::from_str::<Manifest>(&text)
+        && have.version == want.version
+        && have.datasets == want.datasets
+        && have.rows == want.rows
+        && have.cols == want.cols
+        && have.chunk == want.chunk
+        && have.deflate_level == want.deflate_level
+        && have.files == want.files
+        && want.files.iter().all(|(_, f)| dir.join(f).exists())
+    {
+        return Ok(false);
+    }
+    std::fs::create_dir_all(dir)?;
+    // A stale manifest must not survive a half-written regeneration.
+    let _ = std::fs::remove_file(&manifest_path);
+    let n = want.rows * want.cols;
+    for (layout, file) in &want.files {
+        // One layout at a time keeps the peak memory to about twice one
+        // file's decoded size.
+        let mut b = FileBuilder::new();
+        for k in 0..want.datasets {
+            let ds = b.create_dataset(&format!("d{k:02}"));
+            ds.with_f32_data(&dataset_values(k, n))
+                .with_shape(&[want.rows, want.cols]);
+            if layout == "deflate" {
+                ds.with_chunks(&[CHUNK.min(want.rows), CHUNK])
+                    .with_deflate(DEFLATE_LEVEL);
+            }
+        }
+        b.write(dir.join(file)).map_err(std::io::Error::other)?;
+    }
+    std::fs::write(
+        &manifest_path,
+        serde_json::to_string_pretty(want).map_err(std::io::Error::other)?,
+    )?;
+    Ok(true)
+}
+
+fn slab_offsets(seed: u64, count: usize, rows: u64, cols: u64, slab: u64) -> Vec<(u64, u64)> {
+    let mut s = seed;
+    (0..count)
+        .map(|_| {
+            let r = splitmix64(&mut s) % (rows - slab + 1);
+            let c = splitmix64(&mut s) % (cols - slab + 1);
+            (r, c)
+        })
+        .collect()
+}
+
+/// Warm the page cache by reading every byte of `path`.
+fn warm(path: &Path) -> std::io::Result<()> {
+    let mut f = std::fs::File::open(path)?;
+    std::io::copy(&mut f, &mut std::io::sink())?;
+    Ok(())
+}
+
+/// Ask the kernel to drop `path`'s pages from the page cache.
+fn evict(path: &Path) -> std::io::Result<()> {
+    use std::os::fd::AsRawFd;
+    let f = std::fs::File::open(path)?;
+    // SAFETY: plain syscall on a valid, open file descriptor.
+    let rc = unsafe { libc::posix_fadvise(f.as_raw_fd(), 0, 0, libc::POSIX_FADV_DONTNEED) };
+    if rc != 0 {
+        return Err(std::io::Error::from_raw_os_error(rc));
+    }
+    Ok(())
+}
+
+#[derive(Serialize)]
+struct Row {
+    layout: String,
+    mode: String,
+    threads: usize,
+    /// Decoded (selected) bytes read per repetition.
+    bytes: u64,
+    times_s: Vec<f64>,
+    median_s: f64,
+    mb_s: f64,
+    /// `mb_s / (threads * mb_s at threads = 1)`; null without a 1-thread row.
+    efficiency: Option<f64>,
+}
+
+struct Args {
+    dir: PathBuf,
+    datasets: u64,
+    mib: u64,
+    threads: Vec<usize>,
+    reps: usize,
+    slab: u64,
+    slabs: usize,
+    seed: u64,
+    cold: bool,
+    decode_threads: usize,
+    modes: Vec<String>,
+    layouts: Vec<String>,
+    json: Option<PathBuf>,
+}
+
+const USAGE: &str = "\
+usage: concurrent_read [--dir DIR] [--datasets N] [--mib N] [--threads 1,2,4,8,16]
+                       [--reps N] [--slab N] [--slabs N] [--seed N] [--cold]
+                       [--decode-threads N] [--modes distinct,same]
+                       [--layouts deflate,contiguous] [--json FILE]";
+
+fn parse_list<T: std::str::FromStr>(s: &str) -> Result<Vec<T>, String> {
+    s.split(',')
+        .map(|x| x.trim().parse().map_err(|_| format!("bad list item {x:?}")))
+        .collect()
+}
+
+fn parse_args() -> Result<Args, String> {
+    let mut a = Args {
+        dir: PathBuf::from("concurrent-read-data"),
+        datasets: 64,
+        mib: 64,
+        threads: vec![1, 2, 4, 8, 16],
+        reps: 3,
+        slab: 256,
+        slabs: 1024,
+        seed: 42,
+        cold: false,
+        decode_threads: 0,
+        modes: vec!["distinct".into(), "same".into()],
+        layouts: LAYOUTS.iter().map(|s| s.to_string()).collect(),
+        json: None,
+    };
+    let mut it = std::env::args().skip(1);
+    while let Some(flag) = it.next() {
+        if flag == "--cold" {
+            a.cold = true;
+            continue;
+        }
+        if flag == "-h" || flag == "--help" {
+            return Err(USAGE.into());
+        }
+        let v = it.next().ok_or(format!("{flag} needs a value\n{USAGE}"))?;
+        let num = |v: &str| {
+            v.parse::<u64>()
+                .map_err(|_| format!("{flag}: bad number {v:?}"))
+        };
+        match flag.as_str() {
+            "--dir" => a.dir = v.into(),
+            "--datasets" => a.datasets = num(&v)?,
+            "--mib" => a.mib = num(&v)?,
+            "--threads" => a.threads = parse_list(&v)?,
+            "--reps" => a.reps = num(&v)? as usize,
+            "--slab" => a.slab = num(&v)?,
+            "--slabs" => a.slabs = num(&v)? as usize,
+            "--seed" => a.seed = num(&v)?,
+            "--decode-threads" => a.decode_threads = num(&v)? as usize,
+            "--modes" => a.modes = parse_list(&v)?,
+            "--layouts" => a.layouts = parse_list(&v)?,
+            "--json" => a.json = Some(v.into()),
+            _ => return Err(format!("unknown flag {flag}\n{USAGE}")),
+        }
+    }
+    if a.datasets == 0 || a.datasets > 100 {
+        return Err("--datasets must be 1..=100".into());
+    }
+    if a.mib == 0 || a.reps == 0 || a.slabs == 0 || a.threads.contains(&0) {
+        return Err("--mib, --reps, --slabs and every --threads value must be > 0".into());
+    }
+    if a.slab == 0 || a.slab > COLS || a.slab > a.mib * ROWS_PER_MIB {
+        return Err(format!(
+            "--slab must be 1..={}",
+            COLS.min(a.mib * ROWS_PER_MIB)
+        ));
+    }
+    for m in &a.modes {
+        if m != "distinct" && m != "same" {
+            return Err(format!("unknown mode {m:?}"));
+        }
+    }
+    for l in &a.layouts {
+        if !LAYOUTS.contains(&l.as_str()) {
+            return Err(format!("unknown layout {l:?}"));
+        }
+    }
+    Ok(a)
+}
+
+/// One timed repetition: `T` threads on one shared `File`. Returns seconds.
+fn run_once(
+    path: &Path,
+    mode: &str,
+    threads: usize,
+    m: &Manifest,
+    slabs: &[(u64, u64)],
+    slab: u64,
+    verify: bool,
+) -> f64 {
+    let file = File::open(path).expect("open");
+    let barrier = Barrier::new(threads + 1); // + the spawning thread
+    let n = m.rows * m.cols;
+    // Each thread times itself from the barrier; the repetition spans the
+    // earliest start to the latest finish (timing on the spawning thread
+    // instead undercounts whenever it is scheduled after the workers ran).
+    let spans: Vec<(Instant, Instant)> = std::thread::scope(|s| {
+        let handles: Vec<_> = (0..threads)
+            .map(|t| {
+                let (file, barrier) = (&file, &barrier);
+                s.spawn(move || {
+                    barrier.wait();
+                    let start = Instant::now();
+                    match mode {
+                        "distinct" => {
+                            for k in (t as u64..m.datasets).step_by(threads) {
+                                let got = file.dataset(&format!("d{k:02}")).unwrap().read_f32();
+                                let got = got.unwrap();
+                                assert_eq!(got.len() as u64, n);
+                                if verify {
+                                    for i in [0, n / 3, n - 1] {
+                                        assert_eq!(got[i as usize], value(k, i), "d{k:02}[{i}]");
+                                    }
+                                }
+                                std::hint::black_box(got);
+                            }
+                        }
+                        _ => {
+                            let ds = file.dataset("d00").unwrap();
+                            for &(r, c) in slabs.iter().skip(t).step_by(threads) {
+                                let sel = Selection::Hyperslab {
+                                    start: vec![r, c],
+                                    stride: vec![1, 1],
+                                    count: vec![slab, slab],
+                                    block: vec![1, 1],
+                                };
+                                let got = ds.read_f32_selection(&sel).unwrap();
+                                assert_eq!(got.len() as u64, slab * slab);
+                                if verify {
+                                    let last = (r + slab - 1) * m.cols + c + slab - 1;
+                                    assert_eq!(got[0], value(0, r * m.cols + c));
+                                    assert_eq!(*got.last().unwrap(), value(0, last));
+                                }
+                                std::hint::black_box(got);
+                            }
+                        }
+                    }
+                    (start, Instant::now())
+                })
+            })
+            .collect();
+        barrier.wait();
+        handles.into_iter().map(|h| h.join().unwrap()).collect()
+    });
+    let start = spans.iter().map(|s| s.0).min().unwrap();
+    let end = spans.iter().map(|s| s.1).max().unwrap();
+    (end - start).as_secs_f64()
+}
+
+fn median(v: &[f64]) -> f64 {
+    let mut s = v.to_vec();
+    s.sort_by(f64::total_cmp);
+    s[s.len() / 2]
+}
+
+fn hostname() -> String {
+    std::fs::read_to_string("/proc/sys/kernel/hostname")
+        .map(|s| s.trim().to_string())
+        .unwrap_or_else(|_| "unknown".into())
+}
+
+fn main() {
+    let args = match parse_args() {
+        Ok(a) => a,
+        Err(e) => {
+            eprintln!("{e}");
+            std::process::exit(2);
+        }
+    };
+    if cfg!(debug_assertions) {
+        eprintln!("warning: debug build — numbers are meaningless. Use --release.");
+    }
+    if args.decode_threads > 0 {
+        rayon::ThreadPoolBuilder::new()
+            .num_threads(args.decode_threads)
+            .build_global()
+            .expect("configure rayon pool");
+    }
+
+    let manifest = manifest_for(args.datasets, args.mib);
+    let t = Instant::now();
+    match ensure_files(&args.dir, &manifest) {
+        Ok(true) => eprintln!(
+            "generated {} in {:.1} s",
+            args.dir.display(),
+            t.elapsed().as_secs_f64()
+        ),
+        Ok(false) => eprintln!("reusing {}", args.dir.display()),
+        Err(e) => {
+            eprintln!("cannot write test files in {}: {e}", args.dir.display());
+            std::process::exit(1);
+        }
+    }
+    let path_of = |layout: &str| args.dir.join(format!("{layout}.h5"));
+    let slabs = slab_offsets(
+        args.seed,
+        args.slabs,
+        manifest.rows,
+        manifest.cols,
+        args.slab,
+    );
+    let dataset_bytes = manifest.rows * manifest.cols * 4;
+
+    let mut rows: Vec<Row> = Vec::new();
+    println!("| layout | mode | threads | MB/s | efficiency | median s |");
+    println!("|---|---|---:|---:|---:|---:|");
+    for layout in &args.layouts {
+        let path = path_of(layout);
+        // Untimed pass: page cache warm (unless --cold), results checked.
+        if !args.cold {
+            warm(&path).expect("warm page cache");
+        }
+        for mode in &args.modes {
+            run_once(&path, mode, 1, &manifest, &slabs, args.slab, true);
+            let bytes = match mode.as_str() {
+                "distinct" => dataset_bytes * manifest.datasets,
+                _ => args.slab * args.slab * 4 * args.slabs as u64,
+            };
+            let mut base: Option<f64> = None;
+            for &threads in &args.threads {
+                let times: Vec<f64> = (0..args.reps)
+                    .map(|_| {
+                        if args.cold {
+                            evict(&path).expect("posix_fadvise");
+                        }
+                        run_once(&path, mode, threads, &manifest, &slabs, args.slab, false)
+                    })
+                    .collect();
+                let med = median(&times);
+                let mb_s = bytes as f64 / (1 << 20) as f64 / med;
+                if threads == 1 {
+                    base = Some(mb_s);
+                }
+                let efficiency = base.map(|b| mb_s / (threads as f64 * b));
+                println!(
+                    "| {layout} | {mode} | {threads} | {mb_s:.0} | {} | {med:.4} |",
+                    efficiency.map_or("-".into(), |e| format!("{e:.2}"))
+                );
+                rows.push(Row {
+                    layout: layout.clone(),
+                    mode: mode.clone(),
+                    threads,
+                    bytes,
+                    times_s: times,
+                    median_s: med,
+                    mb_s,
+                    efficiency,
+                });
+            }
+        }
+    }
+
+    if let Some(out) = &args.json {
+        let doc = serde_json::json!({
+            "tool": "clawhdf5",
+            "version": env!("CARGO_PKG_VERSION"),
+            "host": hostname(),
+            "cpus": std::thread::available_parallelism().map_or(0, |n| n.get()),
+            "unix_time": std::time::SystemTime::now()
+                .duration_since(std::time::UNIX_EPOCH)
+                .map_or(0, |d| d.as_secs()),
+            "cache": if args.cold { "cold (posix_fadvise DONTNEED before each repetition)" } else { "warm" },
+            "decode_threads": rayon::current_num_threads(),
+            "params": {
+                "datasets": manifest.datasets,
+                "mib": args.mib,
+                "rows": manifest.rows,
+                "cols": manifest.cols,
+                "chunk": manifest.chunk,
+                "deflate_level": manifest.deflate_level,
+                "slab": args.slab,
+                "slabs": args.slabs,
+                "seed": args.seed,
+                "reps": args.reps,
+                "dir": args.dir,
+            },
+            "results": rows,
+        });
+        std::fs::write(out, serde_json::to_string_pretty(&doc).unwrap()).expect("write json");
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn values_are_exact_in_f32() {
+        for k in [0, 7, 63] {
+            for i in [0u64, 1, 4095, 1 << 20, (1 << 24) - 1] {
+                let v = value(k, i);
+                assert_eq!(v, (v as f64) as f32);
+                assert!(v < 32768.0);
+                assert_eq!((v * 256.0).fract(), 0.0);
+            }
+        }
+    }
+
+    /// The h5py script hard-codes this vector to check its splitmix64 port.
+    #[test]
+    fn splitmix64_reference() {
+        let mut s = 42;
+        assert_eq!(splitmix64(&mut s), 0xBDD7_3226_2FEB_6E95);
+    }
+}

crates/clawhdf5-bench/tests/concurrent_read_smoke.rs

@@ -0,0 +1,148 @@
+//! Keeps the concurrent-read harnesses working: runs `concurrent_read`, the
+//! h5py script (threads and processes) and the comparison script end to end
+//! on tiny files. h5py reading the files also checks, element by element at
+//! spot positions, that both harnesses generate the same data and slabs.
+//!
+//! The h5py half is skipped when python3 with h5py is unavailable, unless
+//! `CLAWHDF5_REQUIRE_INTEROP=1`; `CLAWHDF5_PYTHON` picks the interpreter.
+
+use std::path::{Path, PathBuf};
+use std::process::Command;
+
+fn python() -> String {
+    std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
+}
+
+fn interop_required() -> bool {
+    std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
+}
+
+fn python_available() -> bool {
+    Command::new(python())
+        .args(["-c", "import h5py, numpy"])
+        .output()
+        .map(|o| o.status.success())
+        .unwrap_or(false)
+}
+
+fn scripts() -> PathBuf {
+    Path::new(env!("CARGO_MANIFEST_DIR")).join("scripts")
+}
+
+fn run(cmd: &mut Command) -> String {
+    let out = cmd.output().expect("spawn");
+    assert!(
+        out.status.success(),
+        "{cmd:?} failed\nSTDOUT:\n{}\nSTDERR:\n{}",
+        String::from_utf8_lossy(&out.stdout),
+        String::from_utf8_lossy(&out.stderr)
+    );
+    String::from_utf8_lossy(&out.stdout).into_owned()
+}
+
+const SMALL: [&str; 8] = [
+    "--threads",
+    "1,2",
+    "--slabs",
+    "8",
+    "--reps",
+    "1",
+    "--slab",
+    "64",
+];
+
+fn results(path: &Path) -> serde_json::Value {
+    serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap()
+}
+
+#[test]
+fn harnesses_run_end_to_end_on_tiny_files() {
+    let dir = tempfile::TempDir::new().unwrap();
+    let data = dir.path().join("data");
+    let claw = dir.path().join("claw.json");
+
+    let bin = env!("CARGO_BIN_EXE_concurrent_read");
+    run(Command::new(bin)
+        .arg("--dir")
+        .arg(&data)
+        .args(["--datasets", "3", "--mib", "1"])
+        .args(SMALL)
+        .arg("--json")
+        .arg(&claw));
+    // Second run reuses the files (and exercises --cold).
+    let out = Command::new(bin)
+        .arg("--dir")
+        .arg(&data)
+        .args(["--datasets", "3", "--mib", "1", "--cold"])
+        .args(SMALL)
+        .output()
+        .unwrap();
+    assert!(out.status.success());
+    assert!(String::from_utf8_lossy(&out.stderr).contains("reusing"));
+
+    let doc = results(&claw);
+    assert_eq!(doc["tool"], "clawhdf5");
+    // 2 layouts x 2 modes x 2 thread counts.
+    assert_eq!(doc["results"].as_array().unwrap().len(), 8);
+    for r in doc["results"].as_array().unwrap() {
+        assert!(r["mb_s"].as_f64().unwrap() > 0.0, "{r}");
+    }
+
+    if !python_available() {
+        assert!(
+            !interop_required(),
+            "CLAWHDF5_REQUIRE_INTEROP=1 but {} has no h5py",
+            python()
+        );
+        eprintln!("skipping the h5py half: no h5py in {}", python());
+        return;
+    }
+    let mut jsons = vec![claw];
+    for executor in ["threads", "processes"] {
+        let out = dir.path().join(format!("h5py-{executor}.json"));
+        run(Command::new(python())
+            .arg(scripts().join("concurrent_read_h5py.py"))
+            .arg("--dir")
+            .arg(&data)
+            .args(["--executor", executor])
+            .args(SMALL)
+            .arg("--json")
+            .arg(&out));
+        let doc = results(&out);
+        assert_eq!(doc["tool"], format!("h5py-{executor}"));
+        assert_eq!(doc["results"].as_array().unwrap().len(), 8);
+        jsons.push(out);
+    }
+    let table = run(Command::new(python())
+        .arg(scripts().join("compare_concurrent_read.py"))
+        .args(&jsons));
+    assert!(table.contains("| deflate | same | 2 |"), "{table}");
+    assert!(table.contains("clawhdf5 / h5py-processes"), "{table}");
+
+    // A different workload must not be compared.
+    let other = dir.path().join("other.json");
+    run(Command::new(python())
+        .arg(scripts().join("concurrent_read_h5py.py"))
+        .arg("--dir")
+        .arg(&data)
+        .args([
+            "--threads",
+            "1",
+            "--slabs",
+            "4",
+            "--reps",
+            "1",
+            "--slab",
+            "64",
+        ])
+        .arg("--json")
+        .arg(&other));
+    let out = Command::new(python())
+        .arg(scripts().join("compare_concurrent_read.py"))
+        .arg(&jsons[0])
+        .arg(&other)
+        .output()
+        .unwrap();
+    assert!(!out.status.success());
+    assert!(String::from_utf8_lossy(&out.stderr).contains("slabs"));
+}

crates/clawhdf5-format/Cargo.toml

@@ -22,6 +22,13 @@ zstd = { version = "0.13", optional = true }
 blake3 = { version = "1", optional = true }
 libaec-sys = { path = "../libaec-sys", version = "0.1", optional = true }
 pco = { version = "1.0", optional = true }
+# Pure-Rust Zstandard, for the plugin filters that embed zstd (bitshuffle,
+# blosc). The `zstd` feature (filter 32015) links libzstd instead.
+ruzstd = { version = "0.9", optional = true }
+# bzip2 with its default backend, libbz2-rs-sys: a pure-Rust port of
+# libbzip2 (no C is compiled, despite the -sys name).
+bzip2 = { version = "0.6", optional = true }
+snap = { version = "1", optional = true }
 
 [dev-dependencies]
 half = { workspace = true }
@@ -37,7 +44,7 @@ harness = false
 # Deflate backend: `zlib-rs` (pure Rust) by default. `fast-deflate` selects
 # zlib-ng instead (C, built with cmake); flate2 prefers a C zlib whenever one
 # is enabled, so turning it on anywhere in the build overrides the default.
-default = ["std", "checksum", "deflate", "provenance", "zlib-rs", "system-zlib-decompress"]
+default = ["std", "checksum", "deflate", "provenance", "zlib-rs", "system-zlib-decompress", "lzf"]
 std = []
 checksum = []
 deflate = ["flate2"]
@@ -56,6 +63,17 @@ zstd = ["dep:zstd"]
 blake3_hash = ["blake3"]
 szip = ["libaec-sys"]
 pcodec = ["dep:pco"]
+# Plugin filters, pure Rust. LZF (32000) is h5py's built-in compression; it
+# has no dependencies, so it is on by default.
+lzf = []
+# Bitshuffle (32008), with its LZ4 and Zstandard modes.
+bitshuffle = ["lz4_flex", "ruzstd"]
+# bzip2 (307).
+bzip2 = ["dep:bzip2", "std"]
+# Blosc 1 (32001) with its BloscLZ, LZ4, Snappy, Zlib and Zstandard codecs.
+blosc = ["lz4_flex", "ruzstd", "snap", "deflate", "std"]
+# Every plugin filter above.
+plugin-filters = ["lzf", "bitshuffle", "bzip2", "blosc"]
 
 [[bench]]
 name = "parallel_decompress_bench"

crates/clawhdf5-format/src/attribute.rs

@@ -97,7 +97,7 @@ impl AttributeMessage {
             return Ok(Cow::Borrowed(bytes));
         }
         let (file_data, offset_size) = file.ok_or(FormatError::UnresolvedSharedMessage)?;
-        let shared_ref = shared_message::parse_shared_ref(bytes, offset_size)?;
+        let shared_ref = shared_message::parse_shared_ref_sized(bytes, offset_size, length_size)?;
         shared_message::resolve_shared_message(
             file_data,
             &shared_ref,
@@ -362,6 +362,18 @@ fn extract_name(bytes: &[u8]) -> String {
     String::from_utf8_lossy(&bytes[..end]).into_owned()
 }
 
+/// An attribute's datatype gets libhdf5's extra check for a header without
+/// a checksum (see [`Datatype::check_unused_bits`]).
+fn check_in_header(
+    attr: AttributeMessage,
+    header: &ObjectHeader,
+) -> Result<AttributeMessage, FormatError> {
+    if header.version == 1 {
+        attr.datatype.check_unused_bits()?;
+    }
+    Ok(attr)
+}
+
 /// Extract all attribute messages from an object header.
 pub fn extract_attributes(
     header: &ObjectHeader,
@@ -371,7 +383,7 @@ pub fn extract_attributes(
     for msg in &header.messages {
         if msg.msg_type == MessageType::Attribute {
             let attr = AttributeMessage::parse(&msg.data, length_size)?;
-            attrs.push(attr);
+            attrs.push(check_in_header(attr, header)?);
         }
     }
     Ok(attrs)
@@ -394,42 +406,81 @@ pub fn find_attribute<'a>(
 ///
 /// Use this instead of `extract_attributes` when reading files that may use dense storage
 /// (e.g., objects with many attributes, typically >8).
+///
+/// Fails if any attribute cannot be read; see [`extract_attributes_tolerant`]
+/// to read the others.
 pub fn extract_attributes_full(
     file_data: &[u8],
     header: &ObjectHeader,
     offset_size: u8,
     length_size: u8,
+) -> Result<Vec<AttributeMessage>, FormatError> {
+    extract_attributes_with(file_data, header, offset_size, length_size, &mut Err)
+}
+
+/// Like [`extract_attributes_full`], but an attribute that cannot be read
+/// (a corrupt or unsupported attribute message, or a heap object that cannot
+/// be located) is left out and its error returned alongside the attributes
+/// that could be read, instead of failing them all.
+///
+/// Errors in the structures that index the attributes (the Attribute Info
+/// message, the dense-storage heap header or B-tree) still fail the call:
+/// then it is unknown which attributes exist at all.
+pub fn extract_attributes_tolerant(
+    file_data: &[u8],
+    header: &ObjectHeader,
+    offset_size: u8,
+    length_size: u8,
+) -> Result<(Vec<AttributeMessage>, Vec<FormatError>), FormatError> {
+    let mut errors = Vec::new();
+    let attrs = extract_attributes_with(file_data, header, offset_size, length_size, &mut |e| {
+        errors.push(e);
+        Ok(())
+    })?;
+    Ok((attrs, errors))
+}
+
+/// Read every attribute; each one that fails goes to `on_error`, which
+/// either stops the read (returns the error) or skips that attribute.
+fn extract_attributes_with(
+    file_data: &[u8],
+    header: &ObjectHeader,
+    offset_size: u8,
+    length_size: u8,
+    on_error: &mut dyn FnMut(FormatError) -> Result<(), FormatError>,
 ) -> Result<Vec<AttributeMessage>, FormatError> {
     let mut attrs = Vec::new();
 
     // Collect compact attributes (inline in OH)
     for msg in &header.messages {
         if msg.msg_type == MessageType::Attribute {
-            if shared_message::is_shared(msg.flags) {
+            let attr = if shared_message::is_shared(msg.flags) {
                 // Shared attribute: resolve the reference to get actual attribute data
-                let shared_ref = shared_message::parse_shared_ref(&msg.data, offset_size)?;
+                shared_message::parse_shared_ref_sized(&msg.data, offset_size, length_size)
-                let resolved_data = shared_message::resolve_shared_message(
+                    .and_then(|shared_ref| {
+                        shared_message::resolve_shared_message(
                             file_data,
                             &shared_ref,
                             MessageType::Attribute,
                             offset_size,
                             length_size,
-                )?;
+                        )
-                let attr = AttributeMessage::parse_in_file(
+                    })
-                    &resolved_data,
+                    .and_then(|resolved| {
+                        AttributeMessage::parse_in_file(
+                            &resolved,
                             file_data,
                             offset_size,
                             length_size,
-                )?;
+                        )
-                attrs.push(attr);
+                    })
             } else {
-                let attr = AttributeMessage::parse_in_file(
+                AttributeMessage::parse_in_file(&msg.data, file_data, offset_size, length_size)
-                    &msg.data,
+            };
-                    file_data,
+            let attr = attr.and_then(|a| check_in_header(a, header));
-                    offset_size,
+            match attr {
-                    length_size,
+                Ok(attr) => attrs.push(attr),
-                )?;
+                Err(e) => on_error(e)?,
-                attrs.push(attr);
             }
         }
     }
@@ -439,9 +490,15 @@ pub fn extract_attributes_full(
     if let Some(info) = attr_info
         && let Some(fh_addr) = info.fractal_heap_address
     {
-        let dense_attrs =
+        extract_dense_attributes(
-            extract_dense_attributes(file_data, &info, fh_addr, offset_size, length_size)?;
+            file_data,
-        attrs.extend(dense_attrs);
+            &info,
+            fh_addr,
+            offset_size,
+            length_size,
+            &mut attrs,
+            on_error,
+        )?;
     }
 
     Ok(attrs)
@@ -468,7 +525,9 @@ fn extract_dense_attributes(
     fh_addr: u64,
     offset_size: u8,
     length_size: u8,
-) -> Result<Vec<AttributeMessage>, FormatError> {
+    attrs: &mut Vec<AttributeMessage>,
+    on_error: &mut dyn FnMut(FormatError) -> Result<(), FormatError>,
+) -> Result<(), FormatError> {
     // Parse fractal heap
     let fh = FractalHeapHeader::parse(file_data, fh_addr as usize, offset_size, length_size)?;
 
@@ -482,28 +541,32 @@ fn extract_dense_attributes(
     let btree_hdr = BTreeV2Header::parse(file_data, btree_addr as usize, offset_size, length_size)?;
     let records = collect_btree_v2_records(file_data, &btree_hdr, offset_size, length_size)?;
 
-    let mut attrs = Vec::new();
     for record in &records {
         // Per HDF5 spec, both type 8 and type 9 records start with heap_id:
         //   Type 8: heap_id(8) + msg_flags(1) + creation_order(4) + hash(4)
         //   Type 9: heap_id(8) + msg_flags(1) + creation_order(4)
-        let id_offset = 0;
+        let id_len = fh.heap_id_length as usize;
-
+        let Some(id_bytes) = record.data.get(..id_len) else {
-        if record.data.len() < id_offset + fh.heap_id_length as usize {
+            on_error(FormatError::UnexpectedEof {
+                expected: id_len,
+                available: record.data.len(),
+            })?;
             continue;
-        }
+        };
-        let id_bytes = &record.data[id_offset..id_offset + fh.heap_id_length as usize];
-
-        // Read attribute message from fractal heap
-        let attr_data = fh.read_managed_object(file_data, id_bytes, offset_size)?;
 
         // The data in the heap is a complete attribute message
-        let attr =
+        let attr = fh
-            AttributeMessage::parse_in_file(&attr_data, file_data, offset_size, length_size)?;
+            .read_managed_object(file_data, id_bytes, offset_size)
-        attrs.push(attr);
+            .and_then(|attr_data| {
+                AttributeMessage::parse_in_file(&attr_data, file_data, offset_size, length_size)
+            });
+        match attr {
+            Ok(attr) => attrs.push(attr),
+            Err(e) => on_error(e)?,
+        }
     }
 
-    Ok(attrs)
+    Ok(())
 }
 
 #[cfg(test)]
@@ -523,7 +586,8 @@ mod tests {
 
     /// Build an f64 LE datatype message.
     fn build_f64_dt() -> Vec<u8> {
-        let mut buf = build_dt_header(1, 1, [0x00, 0x00, 0x02], 8);
+        // Sign bit 63 (bits 8-15 of the class bits).
+        let mut buf = build_dt_header(1, 1, [0x20, 63, 0x00], 8);
         let mut props = [0u8; 12];
         props[2..4].copy_from_slice(&64u16.to_le_bytes()); // bit_precision
         props[4] = 52; // exp_location

crates/clawhdf5-format/src/btree_v2.rs

@@ -323,39 +323,21 @@ fn collect_internal_records(
     let records_start = pos;
     pos += records_total;
 
-    // Compute sizes for child pointers
+    // Child pointer layout, as libhdf5 computes it (H5B2__hdr_init): the
-    // max_records at child depth - for variable-width nrec encoding
+    // child's record count is always encoded in the width needed for a
+    // *leaf's* maximum, and — below the first internal level — the child
+    // subtree's total record count in the width needed for the most records
+    // a subtree of that depth can hold.
     let child_depth = depth - 1;
-    let max_nrec_child = if child_depth == 0 {
+    let nrec_width = bytes_for_max_records(max_leaf_nrec);
-        max_leaf_nrec
-    } else {
-        // For internal nodes at child_depth, the true max_nrec depends on the
-        // node size, record size, and the recursive width of child pointer
-        // entries (which themselves depend on max_nrec at deeper levels).
-        // Computing the exact value requires iterating from the leaf level
-        // upward, as described in the HDF5 spec (III.A.2 "Computing the Size
-        // of B-tree Nodes").
-        //
-        // We use `max_leaf_nrec * 2` as a conservative upper bound. This
-        // over-estimates the nrec encoding width, which means we may read
-        // slightly more bytes per child pointer than strictly necessary, but
-        // never fewer. The over-read bytes are harmless because we only
-        // decode `num_records` entries (the actual count from the node header).
-        //
-        // Known limitation: for very deep trees (depth > 3) with small record
-        // sizes, the true max could exceed this estimate, causing us to
-        // under-allocate the nrec encoding width and misparse child pointers.
-        // In practice, HDF5 B-tree v2 depths rarely exceed 2-3.
-        max_leaf_nrec * 2
-    };
-    let nrec_width = bytes_for_max_records(max_nrec_child);
-
-    // Total records in subtree width (only if depth > 1)
     let total_nrec_width = if depth > 1 {
-        // Width to hold total records in a subtree
+        bytes_for_max_records(cum_max_records(
-        // We compute max possible total records at this subtree depth
+            node_size,
-        let max_total = header_max_total_records(max_leaf_nrec, depth - 1);
+            record_size,
-        bytes_for_max_records(max_total)
+            offset_size,
+            max_leaf_nrec,
+            child_depth,
+        ))
     } else {
         0
     };
@@ -435,14 +417,36 @@ fn collect_internal_records(
     Ok(())
 }
 
-/// Estimate maximum total records at a given depth (for variable-width encoding).
+/// Most records a subtree whose root is at `depth` can hold (libhdf5's
-fn header_max_total_records(max_leaf_nrec: u64, depth: u16) -> u64 {
+/// `cum_max_nrec`): a leaf holds `max_leaf_nrec`; an internal node at depth
-    // Conservative: branching factor * max_leaf at each level
+/// `d` holds `max_nrec(d)` records and `max_nrec(d) + 1` subtrees of depth
-    let mut total = max_leaf_nrec;
+/// `d - 1`, where `max_nrec(d)` is what fits in a node once each record is
-    for _ in 0..depth {
+/// paired with a child pointer of the width depth `d` needs.
-        total = total.saturating_mul(max_leaf_nrec.max(2));
+fn cum_max_records(
+    node_size: u32,
+    record_size: u16,
+    offset_size: u8,
+    max_leaf_nrec: u64,
+    depth: u16,
+) -> u64 {
+    // Internal node overhead: signature(4) + version(1) + type(1) + checksum(4).
+    const PREFIX: u64 = 10;
+    let nrec_width = bytes_for_max_records(max_leaf_nrec) as u64;
+    let mut cum = max_leaf_nrec;
+    let mut cum_width = 0u64;
+    for d in 1..=depth {
+        let ptr = u64::from(offset_size) + nrec_width + if d > 1 { cum_width } else { 0 };
+        let max_nrec = u64::from(node_size)
+            .saturating_sub(PREFIX)
+            .saturating_sub(ptr)
+            / (u64::from(record_size) + ptr).max(1);
+        cum = max_nrec
+            .saturating_add(1)
+            .saturating_mul(cum)
+            .saturating_add(max_nrec);
+        cum_width = bytes_for_max_records(cum) as u64;
     }
-    total
+    cum
 }
 
 #[cfg(test)]
@@ -512,9 +516,15 @@ mod tests {
         child_nrec: u64,
     ) -> Vec<u8> {
         let max_leaf = max_records_leaf(node_size, record_size);
-        let nrec_width = bytes_for_max_records(if depth == 1 { max_leaf } else { max_leaf * 2 });
+        let nrec_width = bytes_for_max_records(max_leaf);
         let total_width = if depth > 1 {
-            bytes_for_max_records(header_max_total_records(max_leaf, depth - 1))
+            bytes_for_max_records(cum_max_records(
+                node_size,
+                record_size,
+                8,
+                max_leaf,
+                depth - 1,
+            ))
         } else {
             0
         };
@@ -673,4 +683,18 @@ mod tests {
         let records = collect_btree_v2_records(&header, &hdr, 8, 8).unwrap();
         assert!(records.is_empty());
     }
+
+    #[test]
+    fn subtree_capacity_matches_libhdf5() {
+        // A link-name index (11-byte records, 512-byte nodes, 8-byte
+        // addresses): libhdf5's H5B2__hdr_init gives 45 records per leaf,
+        // then cum_max_nrec 1 149 at depth 1 and 26 449 at depth 2 — two
+        // bytes of subtree count in a depth-3 root's child pointers, where
+        // leaf_max^3 = 91 125 would need three.
+        let leaf = max_records_leaf(512, 11);
+        assert_eq!(leaf, 45);
+        assert_eq!(cum_max_records(512, 11, 8, leaf, 0), 45);
+        assert_eq!(cum_max_records(512, 11, 8, leaf, 1), 1_149);
+        assert_eq!(cum_max_records(512, 11, 8, leaf, 2), 26_449);
+    }
 }

crates/clawhdf5-format/src/chunked_read.rs

@@ -15,7 +15,7 @@ use crate::datatype::Datatype;
 use crate::error::FormatError;
 use crate::extensible_array::{ExtensibleArrayHeader, read_extensible_array_chunks};
 use crate::filter_pipeline::FilterPipeline;
-use crate::filters::{all_filters_skipped, decompress_chunk_masked};
+use crate::filters::{all_filters_skipped, decompress_chunk_exact};
 use crate::fixed_array::{FixedArrayHeader, read_fixed_array_chunks};
 #[cfg(feature = "std")]
 use std::sync::Arc;
@@ -65,12 +65,13 @@ fn decompress_all_chunks(
         let raw_chunk = &file_data[c_addr..c_addr + size];
 
         let decompressed = if let Some(pl) = pipeline {
-            decompress_chunk_masked(
+            decompress_chunk_exact(
                 raw_chunk,
                 pl,
                 chunk_total_bytes,
                 element_size,
                 chunk_info.filter_mask,
+                &chunk_info.offsets,
             )?
         } else {
             raw_chunk.to_vec()
@@ -148,6 +149,96 @@ pub(crate) fn checked_byte_len(elements: u64, elem_size: usize) -> Result<usize,
         })
 }
 
+/// The spatial chunk dimensions of a chunked layout (`chunk_dimensions` is
+/// the layout message's list: one per dataspace dimension, then the element
+/// size), after the checks libhdf5 makes when it opens a chunked dataset
+/// (`H5D__chunk_init` / `H5D__chunk_set_sizes`): the chunk rank must match
+/// the dataspace's, no chunk dimension may be 0, and a chunk indexed by a
+/// version-1 B-tree (`layout_version` below 4) may not be 4 GiB or more (the
+/// B-tree records chunk sizes in 32 bits; libhdf5: "chunk size must be < 4GB
+/// with v1 b-tree index"). The other chunk indexes allow larger chunks:
+/// HDF5 2.0 writes them with layout version 5. A zero chunk dimension used
+/// to read as all fill values, and a huge one to hang the reader.
+pub(crate) fn chunk_geometry(
+    chunk_dimensions: &[u32],
+    layout_version: u8,
+    dataspace: &Dataspace,
+    elem_size: usize,
+) -> Result<(usize, Vec<usize>), FormatError> {
+    let rank = chunk_dimensions.len().checked_sub(1).ok_or_else(|| {
+        FormatError::InvalidChunkDimensions("chunked layout has no dimensions".into())
+    })?;
+    if dataspace.dimensions.len() != rank {
+        return Err(FormatError::InvalidChunkDimensions(format!(
+            "dimensionality of chunks doesn't match the dataspace (chunk rank {rank}, \
+             dataspace rank {})",
+            dataspace.dimensions.len()
+        )));
+    }
+    let spatial = &chunk_dimensions[..rank];
+    if let Some(d) = spatial.iter().position(|&c| c == 0) {
+        return Err(FormatError::InvalidChunkDimensions(format!(
+            "chunk size must be > 0, dim = {d}"
+        )));
+    }
+    let bytes = spatial
+        .iter()
+        .fold(elem_size as u128, |acc, &c| acc * u128::from(c));
+    if layout_version < 4 && bytes > u128::from(u32::MAX) {
+        return Err(FormatError::InvalidChunkDimensions(format!(
+            "chunk size must be < 4GB with v1 b-tree index (chunk {spatial:?} of {elem_size}-byte elements)"
+        )));
+    }
+    Ok((rank, spatial.iter().map(|&c| c as usize).collect()))
+}
+
+/// The size of one element of `dt` as stored in the file: a
+/// variable-length element is its length (4), a global heap address
+/// (`offset_size`) and an index (4), not the 16 of [`Datatype::type_size`].
+fn stored_element_size(dt: &Datatype, offset_size: u8) -> u64 {
+    match dt {
+        Datatype::VariableLength { .. } => 8 + u64::from(offset_size),
+        Datatype::Array {
+            base_type,
+            dimensions,
+        } => dimensions
+            .iter()
+            .fold(stored_element_size(base_type, offset_size), |acc, &d| {
+                acc.saturating_mul(u64::from(d))
+            }),
+        _ => u64::from(dt.type_size()),
+    }
+}
+
+/// A chunked layout records the element size as its last dimension, and
+/// libhdf5 refuses a dataset whose datatype has another size
+/// (`H5D__chunk_set_sizes`: "stored datatype size in chunk layout does not
+/// match datatype description"). Reading it anyway laid the chunks out with
+/// the wrong element size.
+pub(crate) fn check_chunk_element_size(
+    layout: &DataLayout,
+    datatype: &Datatype,
+    offset_size: u8,
+) -> Result<(), FormatError> {
+    let DataLayout::Chunked {
+        chunk_dimensions, ..
+    } = layout
+    else {
+        return Ok(());
+    };
+    let Some(&stored) = chunk_dimensions.last() else {
+        return Ok(());
+    };
+    let expected = stored_element_size(datatype, offset_size);
+    if u64::from(stored) != expected {
+        return Err(FormatError::InvalidChunkDimensions(format!(
+            "stored datatype size in chunk layout does not match datatype description \
+             (layout {stored} bytes, datatype {expected})"
+        )));
+    }
+    Ok(())
+}
+
 /// Product of chunk dimensions times the element size, overflow-checked.
 pub(crate) fn checked_chunk_byte_len(
     chunk_dims: &[usize],
@@ -222,7 +313,76 @@ pub fn collect_chunk_info(
     offset_size: u8,
     length_size: u8,
 ) -> Result<Vec<ChunkInfo>, FormatError> {
-    collect_chunk_info_inner(file_data, btree_address, ndims, offset_size, length_size, 0)
+    collect_chunk_info_inner(
+        file_data,
+        btree_address,
+        ndims,
+        None,
+        offset_size,
+        length_size,
+        0,
+    )
+}
+
+/// [`collect_chunk_info`] for a layout with these `chunk_dimensions` (the
+/// layout message's list, element size last), checking every key of the
+/// B-tree as libhdf5 does (`H5D__btree_decode_key`): each coordinate offset
+/// must be a multiple of its chunk dimension. That includes the keys that
+/// only bound a node (internal-node keys and each node's final key), which
+/// is where a corrupt chunk dimension shows when the chunks themselves all
+/// start at offset 0 in that dimension (`cve-2018-11205`). A key that fails
+/// ("bad coordinate offset") means a corrupt index or chunk dimension; the
+/// chunks were read at the wrong place, or the dataset read as fill values.
+pub fn collect_chunk_info_checked(
+    file_data: &[u8],
+    btree_address: u64,
+    chunk_dimensions: &[u32],
+    offset_size: u8,
+    length_size: u8,
+) -> Result<Vec<ChunkInfo>, FormatError> {
+    collect_chunk_info_inner(
+        file_data,
+        btree_address,
+        chunk_dimensions.len(),
+        Some(chunk_dimensions),
+        offset_size,
+        length_size,
+        0,
+    )
+}
+
+/// Check one v1 B-tree chunk key's offsets (see
+/// [`collect_chunk_info_checked`]).
+fn check_key_offsets(offsets: &[u64], chunk_dimensions: &[u32]) -> Result<(), FormatError> {
+    for (&offset, &dim) in offsets.iter().zip(chunk_dimensions) {
+        if dim == 0 || offset % u64::from(dim) != 0 {
+            return Err(FormatError::ChunkedReadError(format!(
+                "bad coordinate offset {offsets:?} for chunk dimensions {chunk_dimensions:?}"
+            )));
+        }
+    }
+    Ok(())
+}
+
+/// Read the `ndims` 8-byte offsets of the chunk key at `pos` (after its
+/// chunk size and filter mask) and check them when `chunk_dimensions` is
+/// given.
+fn read_key_offsets(
+    file_data: &[u8],
+    pos: usize,
+    ndims: usize,
+    chunk_dimensions: Option<&[u32]>,
+) -> Result<Vec<u64>, FormatError> {
+    let mut offsets = Vec::with_capacity(ndims);
+    let mut kp = pos + 8;
+    for _ in 0..ndims {
+        offsets.push(read_offset(file_data, kp, CHUNK_KEY_OFFSET_SIZE)?);
+        kp += CHUNK_KEY_OFFSET_SIZE as usize;
+    }
+    if let Some(dims) = chunk_dimensions {
+        check_key_offsets(&offsets, dims)?;
+    }
+    Ok(offsets)
 }
 
 /// Width of each chunk offset in a v1 chunk B-tree key, independent of the
@@ -237,6 +397,7 @@ fn collect_chunk_info_inner(
     file_data: &[u8],
     btree_address: u64,
     ndims: usize,
+    chunk_dimensions: Option<&[u32]>,
     offset_size: u8,
     _length_size: u8,
     depth: usize,
@@ -296,12 +457,7 @@ fn collect_chunk_info_inner(
                 file_data[pos + 6],
                 file_data[pos + 7],
             ]);
-            let mut offsets = Vec::with_capacity(ndims);
+            let offsets = read_key_offsets(file_data, pos, ndims, chunk_dimensions)?;
-            let mut kp = pos + 8;
-            for _ in 0..ndims {
-                offsets.push(read_offset(file_data, kp, CHUNK_KEY_OFFSET_SIZE)?);
-                kp += CHUNK_KEY_OFFSET_SIZE as usize;
-            }
             pos += key_size;
 
             // Parse child address
@@ -315,7 +471,8 @@ fn collect_chunk_info_inner(
                 address,
             });
         }
-        // Skip final key
+        // The final key only bounds the node; libhdf5 still checks it.
+        read_key_offsets(file_data, pos, ndims, chunk_dimensions)?;
         Ok(chunks)
     } else {
         // Internal node: recurse into children
@@ -324,11 +481,13 @@ fn collect_chunk_info_inner(
 
         let mut child_addrs = Vec::with_capacity(entries_used);
         for _ in 0..entries_used {
-            pos += key_size; // skip key
+            read_key_offsets(file_data, pos, ndims, chunk_dimensions)?;
+            pos += key_size;
             let child_addr = read_offset(file_data, pos, offset_size)?;
             child_addrs.push(child_addr);
             pos += os;
         }
+        read_key_offsets(file_data, pos, ndims, chunk_dimensions)?;
 
         let mut all_chunks = Vec::new();
         for child_addr in child_addrs {
@@ -336,6 +495,7 @@ fn collect_chunk_info_inner(
                 file_data,
                 child_addr,
                 ndims,
+                chunk_dimensions,
                 offset_size,
                 _length_size,
                 depth + 1,
@@ -549,30 +709,13 @@ pub fn list_chunks(
         .ok_or_else(|| FormatError::ChunkedReadError("no address for chunked layout".into()))?;
 
     // Both v3 and v4 include element size as last dim (rank+1)
-    let ndims = chunk_dimensions.len();
+    let (rank, chunk_dims) = chunk_geometry(chunk_dimensions, version, dataspace, elem_size)?;
-    let rank = ndims
-        .checked_sub(1)
-        .ok_or_else(|| FormatError::ChunkedReadError("chunked layout has no dimensions".into()))?;
-    let chunk_dims: Vec<usize> = chunk_dimensions[..rank]
-        .iter()
-        .map(|&d| d as usize)
-        .collect();
-
     let ds_dims: Vec<usize> = dataspace.dimensions.iter().map(|&d| d as usize).collect();
-    if ds_dims.len() != rank {
-        return Err(FormatError::ChunkedReadError(format!(
-            "rank mismatch: dataspace has {} dims, layout has {} chunk dims (rank={})",
-            ds_dims.len(),
-            chunk_dimensions.len(),
-            rank
-        )));
-    }
 
     // Collect chunks based on version and index type
     let mut chunks = match (version, chunk_index_type) {
         (3, _) => {
-            let ndims = chunk_dimensions.len(); // rank+1
+            collect_chunk_info_checked(file_data, addr, chunk_dimensions, offset_size, length_size)?
-            collect_chunk_info(file_data, addr, ndims, offset_size, length_size)?
         }
         (4, Some(1)) => {
             // Single chunk — one chunk covering the entire dataset
@@ -679,6 +822,7 @@ pub fn read_chunked_data(
     offset_size: u8,
     length_size: u8,
 ) -> Result<Vec<u8>, FormatError> {
+    check_chunk_element_size(layout, datatype, offset_size)?;
     let elem_size = datatype.type_size() as usize;
     let (chunks, chunk_dims) = list_chunks(
         file_data,
@@ -802,12 +946,13 @@ pub fn read_chunked_data_cached(
     length_size: u8,
     cache: &ChunkCache,
 ) -> Result<Vec<u8>, FormatError> {
-    let (chunk_dimensions, addr_opt) = match layout {
+    let (chunk_dimensions, version, addr_opt) = match layout {
         DataLayout::Chunked {
             chunk_dimensions,
+            version,
             btree_address,
             ..
-        } => (chunk_dimensions, *btree_address),
+        } => (chunk_dimensions, *version, *btree_address),
         _ => {
             return Err(FormatError::ChunkedReadError(
                 "expected chunked layout".into(),
@@ -818,25 +963,10 @@ pub fn read_chunked_data_cached(
     let addr = addr_opt
         .ok_or_else(|| FormatError::ChunkedReadError("no address for chunked layout".into()))?;
 
+    check_chunk_element_size(layout, datatype, offset_size)?;
     let elem_size = datatype.type_size() as usize;
-    let ndims = chunk_dimensions.len();
+    let (rank, chunk_dims) = chunk_geometry(chunk_dimensions, version, dataspace, elem_size)?;
-    let rank = ndims
-        .checked_sub(1)
-        .ok_or_else(|| FormatError::ChunkedReadError("chunked layout has no dimensions".into()))?;
-    let chunk_dims: Vec<usize> = chunk_dimensions[..rank]
-        .iter()
-        .map(|&d| d as usize)
-        .collect();
-
     let ds_dims: Vec<usize> = dataspace.dimensions.iter().map(|&d| d as usize).collect();
-    if ds_dims.len() != rank {
-        return Err(FormatError::ChunkedReadError(format!(
-            "rank mismatch: dataspace has {} dims, layout has {} chunk dims (rank={})",
-            ds_dims.len(),
-            chunk_dimensions.len(),
-            rank
-        )));
-    }
 
     // The per-file cache is shared across datasets (and threads); every
     // lookup is keyed by this dataset's chunk-index address, so another
@@ -932,12 +1062,13 @@ pub fn read_chunked_data_cached(
     let cache_them = total_bytes <= cache.max_bytes();
     if let Some(pl) = pipeline {
         let decode = |c: &&ChunkInfo| -> Result<Vec<u8>, FormatError> {
-            decompress_chunk_masked(
+            decompress_chunk_exact(
                 raw_bytes(c)?,
                 pl,
                 chunk_total_bytes,
                 elem_size as u32,
                 c.filter_mask,
+                &c.offsets,
             )
         };
         for batch in misses.chunks(DECODE_BATCH) {
@@ -1123,12 +1254,13 @@ pub fn read_chunked_data_sweep(
     cache: &ChunkCache,
     sweep: &mut SweepContext,
 ) -> Result<Vec<u8>, FormatError> {
-    let (chunk_dimensions, addr_opt) = match layout {
+    let (chunk_dimensions, version, addr_opt) = match layout {
         DataLayout::Chunked {
             chunk_dimensions,
+            version,
             btree_address,
             ..
-        } => (chunk_dimensions, *btree_address),
+        } => (chunk_dimensions, *version, *btree_address),
         _ => {
             return Err(FormatError::ChunkedReadError(
                 "expected chunked layout".into(),
@@ -1139,25 +1271,10 @@ pub fn read_chunked_data_sweep(
     let addr = addr_opt
         .ok_or_else(|| FormatError::ChunkedReadError("no address for chunked layout".into()))?;
 
+    check_chunk_element_size(layout, datatype, offset_size)?;
     let elem_size = datatype.type_size() as usize;
-    let ndims = chunk_dimensions.len();
+    let (rank, chunk_dims) = chunk_geometry(chunk_dimensions, version, dataspace, elem_size)?;
-    let rank = ndims
-        .checked_sub(1)
-        .ok_or_else(|| FormatError::ChunkedReadError("chunked layout has no dimensions".into()))?;
-    let chunk_dims: Vec<usize> = chunk_dimensions[..rank]
-        .iter()
-        .map(|&d| d as usize)
-        .collect();
-
     let ds_dims: Vec<usize> = dataspace.dimensions.iter().map(|&d| d as usize).collect();
-    if ds_dims.len() != rank {
-        return Err(FormatError::ChunkedReadError(format!(
-            "rank mismatch: dataspace has {} dims, layout has {} chunk dims (rank={})",
-            ds_dims.len(),
-            chunk_dimensions.len(),
-            rank
-        )));
-    }
 
     // The per-file cache is shared across datasets (and threads); every
     // lookup is keyed by this dataset's chunk-index address, so another
@@ -1217,12 +1334,13 @@ pub fn read_chunked_data_sweep(
             ensure_len(file_data, c_addr, size)?;
             let raw_chunk = &file_data[c_addr..c_addr + size];
             let dec = if let Some(pl) = pipeline {
-                decompress_chunk_masked(
+                decompress_chunk_exact(
                     raw_chunk,
                     pl,
                     chunk_total_bytes,
                     elem_size as u32,
                     chunk_info.filter_mask,
+                    &coord,
                 )?
             } else {
                 raw_chunk.to_vec()
@@ -1275,12 +1393,13 @@ pub fn read_chunked_data_indexed(
     length_size: u8,
     cache: &ChunkCache,
 ) -> Result<Vec<u8>, FormatError> {
-    let (chunk_dimensions, addr_opt) = match layout {
+    let (chunk_dimensions, version, addr_opt) = match layout {
         DataLayout::Chunked {
             chunk_dimensions,
+            version,
             btree_address,
             ..
-        } => (chunk_dimensions, *btree_address),
+        } => (chunk_dimensions, *version, *btree_address),
         _ => {
             return Err(FormatError::ChunkedReadError(
                 "expected chunked layout".into(),
@@ -1291,25 +1410,10 @@ pub fn read_chunked_data_indexed(
     let addr = addr_opt
         .ok_or_else(|| FormatError::ChunkedReadError("no address for chunked layout".into()))?;
 
+    check_chunk_element_size(layout, datatype, offset_size)?;
     let elem_size = datatype.type_size() as usize;
-    let ndims = chunk_dimensions.len();
+    let (rank, chunk_dims) = chunk_geometry(chunk_dimensions, version, dataspace, elem_size)?;
-    let rank = ndims
-        .checked_sub(1)
-        .ok_or_else(|| FormatError::ChunkedReadError("chunked layout has no dimensions".into()))?;
-    let chunk_dims: Vec<usize> = chunk_dimensions[..rank]
-        .iter()
-        .map(|&d| d as usize)
-        .collect();
-
     let ds_dims: Vec<usize> = dataspace.dimensions.iter().map(|&d| d as usize).collect();
-    if ds_dims.len() != rank {
-        return Err(FormatError::ChunkedReadError(format!(
-            "rank mismatch: dataspace has {} dims, layout has {} chunk dims (rank={})",
-            ds_dims.len(),
-            chunk_dimensions.len(),
-            rank
-        )));
-    }
 
     // Chunk index and assembly plan for this dataset, built on first access
     // and kept per dataset (keyed by chunk-index address) in the shared cache.
@@ -1346,12 +1450,13 @@ pub fn read_chunked_data_indexed(
             ensure_len(file_data, c_addr, size)?;
             let raw_chunk = &file_data[c_addr..c_addr + size];
             let decompressed = if let Some(pl) = pipeline {
-                decompress_chunk_masked(
+                decompress_chunk_exact(
                     raw_chunk,
                     pl,
                     chunk_total_bytes,
                     elem_size as u32,
                     *filter_mask,
+                    coord,
                 )?
             } else {
                 raw_chunk.to_vec()
@@ -1620,11 +1725,12 @@ mod tests {
             write_offset(&mut buf, chunk.address, offset_size);
         }
 
-        // Final key (dummy)
+        // Final key (its offsets must be on the chunk grid, as libhdf5
+        // checks; 0 always is)
         buf.extend_from_slice(&0u32.to_le_bytes()); // chunk_size
         buf.extend_from_slice(&0u32.to_le_bytes()); // filter_mask
         for _ in 0..ndims {
-            write_offset(&mut buf, u64::MAX, 8);
+            write_offset(&mut buf, 0, 8);
         }
 
         buf
@@ -1632,6 +1738,48 @@ mod tests {
 
     // --- ChunkInfo collection tests ---
 
+    #[test]
+    fn checked_collection_refuses_keys_off_the_chunk_grid() {
+        let chunk = |offsets: Vec<u64>, address| ChunkInfo {
+            chunk_size: 80,
+            filter_mask: 0,
+            offsets,
+            address,
+        };
+        let good =
+            build_chunk_btree_leaf(&[chunk(vec![0, 0], 0x100), chunk(vec![10, 0], 0x200)], 2, 8);
+        assert_eq!(
+            collect_chunk_info_checked(&good, 0, &[10, 8], 8, 8)
+                .unwrap()
+                .len(),
+            2
+        );
+        // A chunk key off the grid.
+        let bad =
+            build_chunk_btree_leaf(&[chunk(vec![0, 0], 0x100), chunk(vec![7, 0], 0x200)], 2, 8);
+        assert!(collect_chunk_info(&bad, 0, 2, 8, 8).is_ok());
+        assert!(matches!(
+            collect_chunk_info_checked(&bad, 0, &[10, 8], 8, 8),
+            Err(FormatError::ChunkedReadError(m)) if m.starts_with("bad coordinate offset")
+        ));
+        // cve-2018-11205: the chunks all start at 0 in dimension 1, and only
+        // the node's final key shows the chunk dimension is wrong.
+        let mut two_d = build_chunk_btree_leaf(
+            &[chunk(vec![0, 0, 0], 0x100), chunk(vec![10, 0, 0], 0x200)],
+            3,
+            8,
+        );
+        // Final key: (20, 20, 0), the end of a 20 x 20 dataset.
+        let final_key = two_d.len() - 24;
+        two_d[final_key..final_key + 8].copy_from_slice(&20u64.to_le_bytes());
+        two_d[final_key + 8..final_key + 16].copy_from_slice(&20u64.to_le_bytes());
+        assert!(collect_chunk_info_checked(&two_d, 0, &[10, 20, 4], 8, 8).is_ok());
+        assert!(matches!(
+            collect_chunk_info_checked(&two_d, 0, &[10, 32788, 4], 8, 8),
+            Err(FormatError::ChunkedReadError(m)) if m.starts_with("bad coordinate offset [20, 20, 0]")
+        ));
+    }
+
     #[test]
     fn collect_two_chunks_from_leaf() {
         let ndims = 2; // rank+1 for 1D dataset
@@ -1750,6 +1898,51 @@ mod tests {
     use crate::dataspace::{Dataspace, DataspaceType};
     use crate::datatype::{Datatype, DatatypeByteOrder};
 
+    #[test]
+    fn chunk_geometry_matches_libhdf5_open_checks() {
+        let space = |dims: &[u64]| Dataspace {
+            space_type: DataspaceType::Simple,
+            rank: dims.len() as u8,
+            dimensions: dims.to_vec(),
+            max_dimensions: None,
+        };
+        for v in [3, 4] {
+            assert_eq!(
+                chunk_geometry(&[4, 5, 8], v, &space(&[10, 10]), 8).unwrap(),
+                (2, vec![4, 5])
+            );
+            // Rank mismatch.
+            assert!(matches!(
+                chunk_geometry(&[4, 8], v, &space(&[10, 10]), 8),
+                Err(FormatError::InvalidChunkDimensions(m)) if m.contains("doesn't match")
+            ));
+            // Zero dimension (a layout built in memory, bypassing the parser).
+            assert!(matches!(
+                chunk_geometry(&[4, 0, 8], v, &space(&[10, 10]), 8),
+                Err(FormatError::InvalidChunkDimensions(m)) if m.contains("must be > 0")
+            ));
+            assert!(chunk_geometry(&[0xFFFF_FFFF, 1], v, &space(&[10]), 1).is_ok());
+        }
+        // With a v1 B-tree index (layout version 3) the largest chunk is
+        // 4 GiB - 1 bytes: 0x80000000 x 4-byte elements (8 GiB) is refused.
+        // These dims used to hang the reader.
+        assert!(matches!(
+            chunk_geometry(&[0x8000_0000, 4], 3, &space(&[10]), 4),
+            Err(FormatError::InvalidChunkDimensions(m)) if m.contains("4GB with v1 b-tree")
+        ));
+        assert!(matches!(
+            chunk_geometry(&[0xFFFF_FFFF, 0xFFFF_FFFF, 1], 3, &space(&[10, 10]), 1),
+            Err(FormatError::InvalidChunkDimensions(m)) if m.contains("4GB with v1 b-tree")
+        ));
+        // The other chunk indexes (layout version 4, and 5, which is read as
+        // 4) allow chunks of 4 GiB and more; HDF5 2.0 writes them.
+        assert_eq!(
+            chunk_geometry(&[0x2000_0001, 8], 4, &space(&[10]), 8).unwrap(),
+            (1, vec![0x2000_0001])
+        );
+        assert!(chunk_geometry(&[0xFFFF_FFFF, 0xFFFF_FFFF, 1], 4, &space(&[10, 10]), 1).is_ok());
+    }
+
     fn make_f64_type() -> Datatype {
         Datatype::FloatingPoint {
             size: 8,
@@ -1863,8 +2056,8 @@ mod tests {
         let file_data = vec![0u8; 64];
         let result = read_chunked_data(&file_data, &layout, &dataspace, &datatype, None, 8, 8);
         assert!(
-            matches!(result, Err(FormatError::ChunkedReadError(_))),
+            matches!(result, Err(FormatError::InvalidChunkDimensions(_))),
-            "expected a clean ChunkedReadError, got {result:?}"
+            "expected a clean InvalidChunkDimensions, got {result:?}"
         );
     }
 

crates/clawhdf5-format/src/chunked_write.rs

@@ -12,8 +12,9 @@ use crate::chunk_grid::ChunkGrid;
 use crate::ea_writer;
 use crate::error::FormatError;
 use crate::filter_pipeline::{
-    FILTER_DEFLATE, FILTER_FLETCHER32, FILTER_LZ4, FILTER_PCODEC, FILTER_PCODEC_NAME,
+    FILTER_BITSHUFFLE, FILTER_BLOSC, FILTER_BZIP2, FILTER_DEFLATE, FILTER_FLETCHER32, FILTER_LZ4,
-    FILTER_SHUFFLE, FILTER_ZSTD, FilterDescription, FilterPipeline,
+    FILTER_LZF, FILTER_PCODEC, FILTER_PCODEC_NAME, FILTER_SHUFFLE, FILTER_ZSTD, FilterDescription,
+    FilterPipeline,
 };
 use crate::filters::compress_chunk;
 /// Round a file offset up to the next cache-line boundary.
@@ -48,6 +49,167 @@ pub struct ChunkOptions {
     /// Pcodec lossless numerical compression. Private, unregistered filter
     /// ID [`FILTER_PCODEC`] (480): only clawhdf5 can read it.
     pub pcodec: bool,
+    /// A plugin compression filter (LZF, ...). Takes priority over the
+    /// codecs above. Each needs its cargo feature to be written.
+    pub plugin: Option<PluginFilter>,
+}
+
+/// A compression filter from the common HDF5 plugin set, written in the
+/// format the libhdf5 plugin (h5py / hdf5plugin) reads.
+#[derive(Debug, Clone, PartialEq, Eq)]
+#[non_exhaustive]
+pub enum PluginFilter {
+    /// LZF (filter 32000), h5py's built-in `compression="lzf"`. Needs the
+    /// `lzf` feature.
+    Lzf,
+    /// Bitshuffle (filter 32008): a bit transpose of each block of
+    /// `block_size` elements (0 = bitshuffle's default, else a multiple of
+    /// 8), optionally compressed. Needs the `bitshuffle` feature.
+    Bitshuffle {
+        /// Block size in elements; 0 for the default.
+        block_size: u32,
+        /// Compression after the transpose.
+        compression: BitshuffleCompression,
+    },
+    /// bzip2 (filter 307) at block size `level` (1-9). Needs the `bzip2`
+    /// feature.
+    Bzip2 {
+        /// Block size 1-9 (9 = hdf5plugin's default).
+        level: u32,
+    },
+    /// Blosc 1 (filter 32001): `codec` at `level` (0-9; 0 stores), after
+    /// `shuffle`. Needs the `blosc` feature.
+    Blosc {
+        /// The codec inside the Blosc frame.
+        codec: BloscCodec,
+        /// Compression level 0-9 (0 stores the data uncompressed).
+        level: u32,
+        /// The shuffle Blosc applies first.
+        shuffle: BloscShuffle,
+    },
+}
+
+/// The codec inside a Blosc frame that clawhdf5 can write. (It reads
+/// BloscLZ too, but cannot write it.)
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum BloscCodec {
+    /// LZ4.
+    Lz4,
+    /// Snappy.
+    Snappy,
+    /// Zlib, at the Blosc level.
+    Zlib,
+    /// Zstandard (clawhdf5's pure-Rust encoder has one level, about zstd 1).
+    Zstd,
+}
+
+/// The shuffle Blosc applies before compressing.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum BloscShuffle {
+    /// None.
+    None,
+    /// Byte shuffle (Blosc's default).
+    Byte,
+    /// Bit shuffle.
+    Bit,
+}
+
+/// What bitshuffle compresses its blocks with.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum BitshuffleCompression {
+    /// Transpose only.
+    None,
+    /// LZ4 (bitshuffle's `cname="lz4"`, the common choice).
+    Lz4,
+    /// Zstandard. clawhdf5's pure-Rust encoder has a single level (about
+    /// zstd's level 1); `level` is recorded in the file for other writers.
+    Zstd {
+        /// Level recorded in `cd_values[5]`.
+        level: u32,
+    },
+}
+
+impl PluginFilter {
+    /// Whether the filter reorders bytes itself, so the automatic shuffle
+    /// pre-filter would only get in its way.
+    fn shuffles_itself(&self) -> bool {
+        match self {
+            PluginFilter::Lzf => false,
+            PluginFilter::Bitshuffle { .. } => true,
+            PluginFilter::Bzip2 { .. } => false,
+            PluginFilter::Blosc { .. } => true,
+        }
+    }
+
+    /// The pipeline entry for this filter. `chunk_bytes` is one chunk's
+    /// uncompressed size (0 if unknown).
+    fn description(&self, element_size: u32, chunk_bytes: u32) -> FilterDescription {
+        match self {
+            // h5py's lzf_set_local: filter version, liblzf version, chunk
+            // size in bytes. Optional, as h5py flags it: a chunk the filter
+            // cannot shrink may then be stored unfiltered.
+            PluginFilter::Lzf => FilterDescription {
+                filter_id: FILTER_LZF,
+                name: Some("lzf".into()),
+                flags: 1,
+                client_data: vec![4, 0x0105, chunk_bytes],
+            },
+            // bshuf_h5_set_local: version 0.4, element size, block size,
+            // compression (0 none, 2 LZ4, 3 Zstandard), Zstandard level.
+            // hdf5-blosc's blosc_set_local: filter revision 2, Blosc format
+            // 2, type size, chunk size, then level, shuffle, compressor.
+            PluginFilter::Blosc {
+                codec,
+                level,
+                shuffle,
+            } => FilterDescription {
+                filter_id: FILTER_BLOSC,
+                name: Some("blosc".into()),
+                flags: 1,
+                client_data: vec![
+                    2,
+                    2,
+                    element_size,
+                    chunk_bytes,
+                    (*level).min(9),
+                    match shuffle {
+                        BloscShuffle::None => 0,
+                        BloscShuffle::Byte => 1,
+                        BloscShuffle::Bit => 2,
+                    },
+                    match codec {
+                        BloscCodec::Lz4 => 1,
+                        BloscCodec::Snappy => 3,
+                        BloscCodec::Zlib => 4,
+                        BloscCodec::Zstd => 5,
+                    },
+                ],
+            },
+            PluginFilter::Bzip2 { level } => FilterDescription {
+                filter_id: FILTER_BZIP2,
+                name: Some("bzip2".into()),
+                flags: 1,
+                client_data: vec![(*level).clamp(1, 9)],
+            },
+            PluginFilter::Bitshuffle {
+                block_size,
+                compression,
+            } => {
+                let mut cd = vec![0, 4, element_size, *block_size];
+                match compression {
+                    BitshuffleCompression::None => cd.push(0),
+                    BitshuffleCompression::Lz4 => cd.push(2),
+                    BitshuffleCompression::Zstd { level } => cd.extend([3, *level]),
+                }
+                FilterDescription {
+                    filter_id: FILTER_BITSHUFFLE,
+                    name: Some("bitshuffle; see https://github.com/kiyo-masui/bitshuffle".into()),
+                    flags: 1,
+                    client_data: cd,
+                }
+            }
+        }
+    }
 }
 
 /// Largest chunk the automatic choice produces, in bytes.
@@ -92,14 +254,33 @@ impl ChunkOptions {
             || self.lz4
             || self.zstd_level.is_some()
             || self.pcodec
+            || self.plugin.is_some()
     }
 
     /// Build a FilterPipeline from the options.
     pub fn build_pipeline(&self, element_size: u32) -> Option<FilterPipeline> {
+        self.build_pipeline_for_chunk(element_size, 0)
+    }
+
+    /// Build a FilterPipeline for chunks of `chunk_bytes` uncompressed bytes
+    /// (0 if unknown). Some plugin filters record the chunk size in their
+    /// client data.
+    pub fn build_pipeline_for_chunk(
+        &self,
+        element_size: u32,
+        chunk_bytes: u32,
+    ) -> Option<FilterPipeline> {
         let mut filters = Vec::new();
 
-        let has_compression =
+        let plugin_shuffles = self
-            self.deflate_level.is_some() || self.zstd_level.is_some() || self.lz4 || self.pcodec;
+            .plugin
+            .as_ref()
+            .is_some_and(PluginFilter::shuffles_itself);
+        let has_compression = self.deflate_level.is_some()
+            || self.zstd_level.is_some()
+            || self.lz4
+            || self.pcodec
+            || (self.plugin.is_some() && !plugin_shuffles);
 
         // Shuffle before compression. Applied if explicitly requested OR if compression
         // is active and the caller hasn't disabled it — matches h5py default behavior
@@ -113,8 +294,11 @@ impl ChunkOptions {
             });
         }
 
-        // Compression filters (mutually exclusive, priority: pcodec > zstd > lz4 > deflate)
+        // Compression filters (mutually exclusive, priority: plugin > pcodec >
-        if self.pcodec {
+        // zstd > lz4 > deflate)
+        if let Some(plugin) = &self.plugin {
+            filters.push(plugin.description(element_size, chunk_bytes));
+        } else if self.pcodec {
             filters.push(FilterDescription {
                 filter_id: FILTER_PCODEC,
                 name: Some(FILTER_PCODEC_NAME.into()),
@@ -383,39 +567,7 @@ fn serialize_v4_single_chunk(
     let ndims = chunk_dims.len() as u8 + 1;
     buf.push(ndims);
 
-    // dim_size_encoded_length: how many bytes per dimension
+    push_v4_chunk_dims(&mut buf, chunk_dims, element_size);
-    // We need to figure out the minimum encoding width
-    let max_dim = chunk_dims
-        .iter()
-        .map(|&d| d as u64)
-        .chain(core::iter::once(element_size as u64))
-        .max()
-        .unwrap_or(1);
-    let dim_encoded_len: u8 = if max_dim <= 0xFF {
-        1
-    } else if max_dim <= 0xFFFF {
-        2
-    } else {
-        4
-    };
-    buf.push(dim_encoded_len);
-
-    // dimension sizes (chunk dims + element size)
-    for &d in chunk_dims {
-        match dim_encoded_len {
-            1 => buf.push(d as u8),
-            2 => buf.extend_from_slice(&(d as u16).to_le_bytes()),
-            4 => buf.extend_from_slice(&d.to_le_bytes()),
-            _ => {}
-        }
-    }
-    // Element size dimension
-    match dim_encoded_len {
-        1 => buf.push(element_size as u8),
-        2 => buf.extend_from_slice(&(element_size as u16).to_le_bytes()),
-        4 => buf.extend_from_slice(&element_size.to_le_bytes()),
-        _ => {}
-    }
 
     // chunk index type = 1 (single chunk)
     buf.push(1);
@@ -465,6 +617,25 @@ fn serialize_v4_fixed_array(
 
 /// The part of a v4 chunked layout message before the chunk index type:
 /// version, class, flags and the chunk dimensions (plus the element size).
+/// Append a v4 layout's dimension width and its dimensions (the chunk
+/// dimensions, then the element size). Each takes the fewest bytes that hold
+/// the largest, as libhdf5 computes it (`H5D__chunk_set_sizes`:
+/// `(log2(dim) + 8) / 8`); HDF5 2.0.0 refuses any other width.
+pub(crate) fn push_v4_chunk_dims(buf: &mut Vec<u8>, chunk_dims: &[u32], element_size: u32) {
+    let max_dim = chunk_dims
+        .iter()
+        .copied()
+        .chain(core::iter::once(element_size))
+        .max()
+        .unwrap_or(1)
+        .max(1);
+    let width = (32 - max_dim.leading_zeros()).div_ceil(8) as usize;
+    buf.push(width as u8);
+    for &d in chunk_dims.iter().chain(core::iter::once(&element_size)) {
+        buf.extend_from_slice(&d.to_le_bytes()[..width]);
+    }
+}
+
 fn layout_v4_chunked_prefix(chunk_dims: &[u32], element_size: u32) -> Vec<u8> {
     let mut buf = Vec::new();
     buf.push(4); // version
@@ -476,35 +647,7 @@ fn layout_v4_chunked_prefix(chunk_dims: &[u32], element_size: u32) -> Vec<u8> {
     let ndims = chunk_dims.len() as u8 + 1;
     buf.push(ndims);
 
-    let max_dim = chunk_dims
+    push_v4_chunk_dims(&mut buf, chunk_dims, element_size);
-        .iter()
-        .map(|&d| d as u64)
-        .chain(core::iter::once(element_size as u64))
-        .max()
-        .unwrap_or(1);
-    let dim_encoded_len: u8 = if max_dim <= 0xFF {
-        1
-    } else if max_dim <= 0xFFFF {
-        2
-    } else {
-        4
-    };
-    buf.push(dim_encoded_len);
-
-    for &d in chunk_dims {
-        match dim_encoded_len {
-            1 => buf.push(d as u8),
-            2 => buf.extend_from_slice(&(d as u16).to_le_bytes()),
-            4 => buf.extend_from_slice(&d.to_le_bytes()),
-            _ => {}
-        }
-    }
-    match dim_encoded_len {
-        1 => buf.push(element_size as u8),
-        2 => buf.extend_from_slice(&(element_size as u16).to_le_bytes()),
-        4 => buf.extend_from_slice(&element_size.to_le_bytes()),
-        _ => {}
-    }
     buf
 }
 
@@ -675,7 +818,12 @@ pub fn precompress_chunks(
     element_size: usize,
     options: &ChunkOptions,
 ) -> Result<PrecompressedChunks, FormatError> {
-    let pipeline = options.build_pipeline(element_size as u32);
+    let chunk_bytes = chunk_dims
+        .iter()
+        .try_fold(element_size as u64, |acc, &d| acc.checked_mul(d))
+        .and_then(|b| u32::try_from(b).ok())
+        .unwrap_or(0);
+    let pipeline = options.build_pipeline_for_chunk(element_size as u32, chunk_bytes);
     let has_filters = pipeline.is_some();
     let pipeline_message = pipeline.as_ref().map(|pl| pl.serialize());
 
@@ -1569,6 +1717,35 @@ mod tests {
         assert_eq!(pl.filters[1].client_data, vec![3]);
     }
 
+    #[test]
+    fn chunk_options_pipeline_lzf() {
+        let options = ChunkOptions {
+            plugin: Some(PluginFilter::Lzf),
+            ..Default::default()
+        };
+        assert!(options.is_chunked());
+        let pl = options.build_pipeline_for_chunk(8, 800).unwrap();
+        assert_eq!(pl.filters.len(), 2);
+        assert_eq!(pl.filters[0].filter_id, FILTER_SHUFFLE);
+        assert_eq!(pl.filters[1].filter_id, FILTER_LZF);
+        assert_eq!(pl.filters[1].client_data, vec![4, 0x0105, 800]);
+    }
+
+    #[test]
+    fn chunk_options_pipeline_bitshuffle_has_no_auto_shuffle() {
+        let options = ChunkOptions {
+            plugin: Some(PluginFilter::Bitshuffle {
+                block_size: 0,
+                compression: BitshuffleCompression::Zstd { level: 5 },
+            }),
+            ..Default::default()
+        };
+        let pl = options.build_pipeline(4).unwrap();
+        assert_eq!(pl.filters.len(), 1);
+        assert_eq!(pl.filters[0].filter_id, FILTER_BITSHUFFLE);
+        assert_eq!(pl.filters[0].client_data, vec![0, 4, 4, 0, 3, 5]);
+    }
+
     #[test]
     fn chunk_options_zstd_priority_over_deflate() {
         let options = ChunkOptions {

crates/clawhdf5-format/src/data_layout.rs

@@ -1,7 +1,7 @@
 //! HDF5 Data Layout message parsing (message type 0x0008).
 
 #[cfg(not(feature = "std"))]
-use alloc::{string::String, vec::Vec};
+use alloc::{format, string::String, vec::Vec};
 
 #[cfg(feature = "std")]
 use std::string::String;
@@ -24,6 +24,34 @@ pub struct VdsMapping {
     pub virtual_selection: Vec<u8>,
 }
 
+/// Most dimensions a layout message can list (libhdf5 `H5O_LAYOUT_NDIMS`):
+/// 32 dataspace dimensions plus the element size.
+const MAX_LAYOUT_NDIMS: usize = 33;
+
+/// libhdf5's checks on a chunked layout message's dimensions
+/// (`H5O__layout_decode`): at most [`MAX_LAYOUT_NDIMS`], no dimension 0, and
+/// before version 4 at least one dataspace dimension plus the element size.
+/// A zero chunk dimension used to read the dataset as all fill values.
+fn check_chunk_dims(dims: Vec<u32>, layout_version: u8) -> Result<Vec<u32>, FormatError> {
+    if dims.len() > MAX_LAYOUT_NDIMS {
+        return Err(FormatError::InvalidChunkDimensions(
+            "dimensionality is too large".into(),
+        ));
+    }
+    if layout_version < 4 && dims.len() < 2 {
+        return Err(FormatError::InvalidChunkDimensions(
+            "bad dimensions for chunked storage".into(),
+        ));
+    }
+    if let Some(u) = dims.iter().position(|&d| d == 0) {
+        return Err(FormatError::InvalidChunkDimensions(format!(
+            "bad chunk dimension value when parsing layout message - chunk dimension must be \
+             positive: mesg->u.chunk.dim[{u}] = 0"
+        )));
+    }
+    Ok(dims)
+}
+
 /// Parsed HDF5 data layout message.
 #[derive(Debug, Clone, PartialEq)]
 pub enum DataLayout {
@@ -45,7 +73,9 @@ pub enum DataLayout {
         chunk_dimensions: Vec<u32>,
         /// B-tree address, or `None` if undefined.
         btree_address: Option<u64>,
-        /// Layout version (3 or 4).
+        /// Layout version (3 or 4). Version 1/2 messages (HDF5 1.4/1.6-era)
+        /// use the same version-1 B-tree chunk index as version 3 and are
+        /// reported as 3.
         version: u8,
         /// Chunk index type (v4 only).
         chunk_index_type: Option<u8>,
@@ -72,21 +102,33 @@ pub enum DataLayout {
     },
 }
 
+/// Version-1 VDS mapping flag: the source file name is stored by an earlier
+/// entry, whose index follows in place of the name.
+const VDS_SOURCE_FILE_SHARED: u8 = 0x01;
+/// Version-1 VDS mapping flag: likewise for the source dataset name.
+const VDS_SOURCE_DSET_SHARED: u8 = 0x02;
+/// Version-1 VDS mapping flag: the source is in the virtual file itself
+/// (`"."`); no file name is stored.
+const VDS_SOURCE_SAME_FILE: u8 = 0x04;
+const VDS_ALL_FLAGS: u8 = VDS_SOURCE_FILE_SHARED | VDS_SOURCE_DSET_SHARED | VDS_SOURCE_SAME_FILE;
+
 /// Parse VDS mappings from global-heap object data.
 ///
 /// The global-heap block holding a VDS mapping list is laid out as
-/// (reverse-engineered and validated against HDF5 2.0):
+/// (`H5D__virtual_store_layout` / `H5D__virtual_load_layout` in libhdf5):
 ///
 /// ```text
 /// version(1) · nused(length_size, LE) · entry[nused] · checksum(4)
 /// ```
 ///
 /// Each entry is:
-/// - source file name — a null-terminated string in **block version 0**; in
+/// - **block version 1 only:** a flags byte. `0x04`: the source is in the
-///   **block version 1** a same-file reference is encoded as a single `0x04`
+///   virtual file itself and no file name is stored; `0x01`/`0x02`: the
-///   marker byte (the source file is the virtual file itself) in place of the
+///   source file/dataset name is that of an earlier entry, whose index
-///   name;
+///   (`length_size` bytes) is stored instead of the name. libhdf5 2.0 writes
-/// - source dataset name (null-terminated string);
+///   version 1 when the file's low version bound is 2.0 and it saves space;
+/// - source file name (null-terminated string, unless flagged above);
+/// - source dataset name (null-terminated string, unless flagged above);
 /// - source selection (serialized `H5S` dataspace selection — self-describing
 ///   in length);
 /// - virtual selection (serialized `H5S` dataspace selection).
@@ -112,7 +154,7 @@ pub fn parse_vds_mappings(
     // `nused` is untrusted; don't pre-allocate from it. Each entry consumes at
     // least a few bytes, so the loop is naturally bounded by the heap data and
     // a bogus `nused` simply errors out on the first short read.
-    let mut mappings = Vec::new();
+    let mut mappings: Vec<VdsMapping> = Vec::new();
     // Reads one self-describing selection at `pos`, returning its raw bytes and
     // advancing past it — bounds-checked so a corrupt selection can't overrun.
     let read_selection = |heap_data: &[u8], pos: &mut usize| -> Result<Vec<u8>, FormatError> {
@@ -132,17 +174,57 @@ pub fn parse_vds_mappings(
         Ok(bytes)
     };
 
-    for _ in 0..nused {
+    if version > 1 {
-        // Source file name (with the version-1 same-file marker handled).
+        return Err(FormatError::ChunkedReadError(
-        let source_file = if version >= 1 && heap_data.get(pos) == Some(&0x04) {
+            "unsupported VDS mapping block version".into(),
+        ));
+    }
+    for i in 0..nused {
+        // Version 1 prefixes each entry with a flags byte; a name may then be
+        // omitted (same file) or replaced by the index of an earlier entry
+        // holding the same name (`H5D__virtual_load_layout`).
+        let flags = if version >= 1 {
+            let f = *heap_data.get(pos).ok_or(FormatError::UnexpectedEof {
+                expected: pos + 1,
+                available: heap_data.len(),
+            })?;
             pos += 1;
+            if f & !VDS_ALL_FLAGS != 0 {
+                return Err(FormatError::ChunkedReadError(
+                    "unknown VDS mapping flags".into(),
+                ));
+            }
+            f
+        } else {
+            0
+        };
+        // Index of an earlier entry, for a shared name.
+        let earlier = |pos: &mut usize| -> Result<usize, FormatError> {
+            let idx = read_length(heap_data, *pos, length_size)?;
+            *pos += ls;
+            if idx >= i {
+                return Err(FormatError::ChunkedReadError(
+                    "VDS mapping shares a name with a later entry".into(),
+                ));
+            }
+            Ok(idx as usize)
+        };
+
+        let source_file = if flags & VDS_SOURCE_SAME_FILE != 0 {
             String::from(".")
+        } else if flags & VDS_SOURCE_FILE_SHARED != 0 {
+            let idx = earlier(&mut pos)?;
+            mappings[idx].source_file.clone()
         } else {
             read_null_terminated_string(heap_data, &mut pos)?
         };
 
-        // Source dataset name.
+        let source_dataset = if flags & VDS_SOURCE_DSET_SHARED != 0 {
-        let source_dataset = read_null_terminated_string(heap_data, &mut pos)?;
+            let idx = earlier(&mut pos)?;
+            mappings[idx].source_dataset.clone()
+        } else {
+            read_null_terminated_string(heap_data, &mut pos)?
+        };
 
         // Source selection, then virtual selection (both self-describing length).
         let source_selection = read_selection(heap_data, &mut pos)?;
@@ -261,6 +343,7 @@ impl DataLayout {
         let layout_class = data[1];
 
         match version {
+            1 | 2 => Self::parse_v1_v2(data, offset_size),
             3 => Self::parse_v3(data, layout_class, offset_size, length_size),
             // v5 (emitted by HDF5 1.14+/2.0 with `libver=latest`) uses the same
             // message structure as v4 — only the version number was bumped.
@@ -269,6 +352,87 @@ impl DataLayout {
         }
     }
 
+    /// Layout message versions 1 and 2 (HDF5 before 1.6.3):
+    ///
+    /// ```text
+    /// version(1) · dimensionality(1) · layout class(1) · reserved(5)
+    /// · address(offset_size)      — contiguous and chunked only
+    /// · dimension sizes(4 × dimensionality)
+    /// · compact data size(4) · compact raw data — compact only
+    /// ```
+    ///
+    /// The dimension sizes are the dataset's (contiguous/compact) or the
+    /// chunk's (chunked) extent plus a trailing element-size dimension, as in
+    /// version 3's chunked form. libhdf5 ignores them for contiguous storage
+    /// and sizes the data from the dataspace; the product of the stored
+    /// dimensions is that same size, and a disagreement (a dimension that was
+    /// truncated to 32 bits) is caught by the reader's size check rather than
+    /// returning wrong data.
+    fn parse_v1_v2(data: &[u8], offset_size: u8) -> Result<DataLayout, FormatError> {
+        ensure_len(data, 0, 8)?;
+        let dimensionality = data[1] as usize;
+        let layout_class = data[2];
+        // H5O_LAYOUT_NDIMS: 32 dataspace dimensions + the element-size one.
+        if dimensionality > 33 {
+            return Err(FormatError::Overflow(format!(
+                "data layout dimensionality {dimensionality} exceeds 33"
+            )));
+        }
+        let mut p = 8;
+        let os = offset_size as usize;
+        let address = match layout_class {
+            1 | 2 => {
+                ensure_len(data, p, os)?;
+                let a = if is_undefined(data, p, offset_size) {
+                    None
+                } else {
+                    Some(read_offset(data, p, offset_size)?)
+                };
+                p += os;
+                a
+            }
+            0 => None,
+            _ => return Err(FormatError::InvalidLayoutClass(layout_class)),
+        };
+        ensure_len(data, p, dimensionality * 4)?;
+        let dims: Vec<u32> = data[p..p + dimensionality * 4]
+            .as_chunks::<4>()
+            .0
+            .iter()
+            .map(|c| u32::from_le_bytes(*c))
+            .collect();
+        p += dimensionality * 4;
+        match layout_class {
+            0 => {
+                ensure_len(data, p, 4)?;
+                let size =
+                    u32::from_le_bytes([data[p], data[p + 1], data[p + 2], data[p + 3]]) as usize;
+                ensure_len(data, p + 4, size)?;
+                Ok(DataLayout::Compact {
+                    data: data[p + 4..p + 4 + size].to_vec(),
+                })
+            }
+            1 => {
+                let size = dims
+                    .iter()
+                    .try_fold(1u64, |acc, &d| acc.checked_mul(d as u64))
+                    .ok_or_else(|| {
+                        FormatError::Overflow(format!("contiguous layout size {dims:?}"))
+                    })?;
+                Ok(DataLayout::Contiguous { address, size })
+            }
+            _ => Ok(DataLayout::Chunked {
+                chunk_dimensions: check_chunk_dims(dims, 2)?,
+                btree_address: address,
+                version: 3,
+                chunk_index_type: None,
+                single_chunk_filtered_size: None,
+                single_chunk_filter_mask: None,
+                dont_filter_partial_edge_chunks: false,
+            }),
+        }
+    }
+
     fn parse_v3(
         data: &[u8],
         layout_class: u8,
@@ -321,7 +485,7 @@ impl DataLayout {
                     p += 4;
                 }
                 Ok(DataLayout::Chunked {
-                    chunk_dimensions,
+                    chunk_dimensions: check_chunk_dims(chunk_dimensions, 3)?,
                     btree_address,
                     version: 3,
                     chunk_index_type: None,
@@ -370,47 +534,40 @@ impl DataLayout {
                 let dimensionality = data[pos + 1] as usize;
                 let dim_size_encoded_length = data[pos + 2] as usize;
                 let mut p = pos + 3;
+                if dimensionality > MAX_LAYOUT_NDIMS {
+                    return Err(FormatError::InvalidChunkDimensions(
+                        "dimensionality is too large".into(),
+                    ));
+                }
 
-                // dimension sizes
+                // Each dimension takes 1 to 8 bytes (libhdf5 writes the
+                // fewest that hold the largest one, so 3, 5, 6 and 7 occur:
+                // a chunk dimension of 70 000 takes 3). libhdf5 refuses 0
+                // and more than 8.
+                if dim_size_encoded_length == 0 || dim_size_encoded_length > 8 {
+                    return Err(FormatError::InvalidChunkDimensions(
+                        "encoded chunk dimension size is too large".into(),
+                    ));
+                }
                 ensure_len(data, p, dimensionality * dim_size_encoded_length)?;
                 let mut chunk_dimensions = Vec::with_capacity(dimensionality);
                 for _ in 0..dimensionality {
-                    let val = match dim_size_encoded_length {
+                    let val = data[p..p + dim_size_encoded_length]
-                        1 => data[p] as u32,
+                        .iter()
-                        2 => u16::from_le_bytes([data[p], data[p + 1]]) as u32,
+                        .rev()
-                        4 => u32::from_le_bytes([data[p], data[p + 1], data[p + 2], data[p + 3]]),
+                        .fold(0u64, |acc, &b| (acc << 8) | u64::from(b));
-                        8 => {
+                    // Chunk dimensions are held as u32; HDF5 2.0 can write
-                            // V4 chunked encodes dimension sizes as 8 bytes, but
+                    // larger ones (layout version 5), which are refused
-                            // our ChunkedStorageV4 stores them as u32. We read only
+                    // rather than truncated.
-                            // the low 4 bytes (little-endian). This silently
+                    let val = u32::try_from(val).map_err(|_| {
-                            // truncates dimensions > 4 GiB, which are not expected
+                        FormatError::InvalidChunkDimensions(format!(
-                            // in practice (HDF5 chunk dimensions are always small).
+                            "chunk dimension {val} is larger than 2^32 - 1, which is not supported"
-                            // If the high bytes are non-zero, the file is malformed
+                        ))
-                            // or uses dimensions we cannot represent.
+                    })?;
-                            let high = u32::from_le_bytes([
-                                data[p + 4],
-                                data[p + 5],
-                                data[p + 6],
-                                data[p + 7],
-                            ]);
-                            if high != 0 {
-                                return Err(FormatError::UnexpectedEof {
-                                    expected: p + 8,
-                                    available: data.len(),
-                                });
-                            }
-                            u32::from_le_bytes([data[p], data[p + 1], data[p + 2], data[p + 3]])
-                        }
-                        _ => {
-                            return Err(FormatError::UnexpectedEof {
-                                expected: p + dim_size_encoded_length,
-                                available: data.len(),
-                            });
-                        }
-                    };
                     chunk_dimensions.push(val);
                     p += dim_size_encoded_length;
                 }
+                let chunk_dimensions = check_chunk_dims(chunk_dimensions, 4)?;
 
                 // chunk index type
                 ensure_len(data, p, 1)?;
@@ -546,6 +703,202 @@ impl DataLayout {
 mod tests {
     use super::*;
 
+    /// Version 1/2 header: version, dimensionality, class, reserved(5).
+    fn v1v2_header(version: u8, ndims: u8, class: u8) -> Vec<u8> {
+        vec![version, ndims, class, 0, 0, 0, 0, 0]
+    }
+
+    #[test]
+    fn v2_compact() {
+        let mut buf = v1v2_header(2, 2, 0);
+        // dims (3 elements of 2 bytes) — no address for compact
+        buf.extend_from_slice(&3u32.to_le_bytes());
+        buf.extend_from_slice(&2u32.to_le_bytes());
+        buf.extend_from_slice(&6u32.to_le_bytes()); // compact size (u32 in v1/v2)
+        buf.extend_from_slice(&[1, 0, 2, 0, 3, 0]);
+        assert_eq!(
+            DataLayout::parse(&buf, 8, 8).unwrap(),
+            DataLayout::Compact {
+                data: vec![1, 0, 2, 0, 3, 0]
+            }
+        );
+    }
+
+    #[test]
+    fn v1_contiguous_size_from_dimensions() {
+        let mut buf = v1v2_header(1, 3, 1);
+        buf.extend_from_slice(&0x800u32.to_le_bytes()); // 4-byte address
+        for d in [10u32, 20, 4] {
+            buf.extend_from_slice(&d.to_le_bytes());
+        }
+        assert_eq!(
+            DataLayout::parse(&buf, 4, 4).unwrap(),
+            DataLayout::Contiguous {
+                address: Some(0x800),
+                size: 800,
+            }
+        );
+    }
+
+    #[test]
+    fn v1_contiguous_undefined_address() {
+        let mut buf = v1v2_header(1, 2, 1);
+        buf.extend_from_slice(&[0xFF; 8]);
+        buf.extend_from_slice(&5u32.to_le_bytes());
+        buf.extend_from_slice(&8u32.to_le_bytes());
+        assert_eq!(
+            DataLayout::parse(&buf, 8, 8).unwrap(),
+            DataLayout::Contiguous {
+                address: None,
+                size: 40,
+            }
+        );
+    }
+
+    #[test]
+    fn v1_chunked_maps_to_btree_v1_index() {
+        let mut buf = v1v2_header(1, 3, 2);
+        buf.extend_from_slice(&0x1234u64.to_le_bytes());
+        for d in [50u32, 50, 4] {
+            buf.extend_from_slice(&d.to_le_bytes());
+        }
+        assert_eq!(
+            DataLayout::parse(&buf, 8, 8).unwrap(),
+            DataLayout::Chunked {
+                chunk_dimensions: vec![50, 50, 4],
+                btree_address: Some(0x1234),
+                version: 3,
+                chunk_index_type: None,
+                single_chunk_filtered_size: None,
+                single_chunk_filter_mask: None,
+                dont_filter_partial_edge_chunks: false,
+            }
+        );
+    }
+
+    /// A v3 chunked layout message with these dims (element size last).
+    fn v3_chunked_msg(dims: &[u32]) -> Vec<u8> {
+        let mut buf = vec![3u8, 2, dims.len() as u8];
+        buf.extend_from_slice(&0x1000u64.to_le_bytes());
+        for d in dims {
+            buf.extend_from_slice(&d.to_le_bytes());
+        }
+        buf
+    }
+
+    #[test]
+    fn chunk_dimensions_are_checked_when_the_layout_is_parsed() {
+        assert!(DataLayout::parse(&v3_chunked_msg(&[4, 4, 8]), 8, 8).is_ok());
+        // A zero chunk dimension used to read as all fill values.
+        let err = DataLayout::parse(&v3_chunked_msg(&[4, 0, 8]), 8, 8).unwrap_err();
+        assert!(
+            matches!(&err, FormatError::InvalidChunkDimensions(m) if m.contains("dim[1] = 0")),
+            "{err:?}"
+        );
+        // Only the element-size dimension: libhdf5 "bad dimensions".
+        assert_eq!(
+            DataLayout::parse(&v3_chunked_msg(&[8]), 8, 8).unwrap_err(),
+            FormatError::InvalidChunkDimensions("bad dimensions for chunked storage".into())
+        );
+        assert_eq!(
+            DataLayout::parse(&v3_chunked_msg(&[1; 34]), 8, 8).unwrap_err(),
+            FormatError::InvalidChunkDimensions("dimensionality is too large".into())
+        );
+        // v1/v2 and v4 messages get the zero check too.
+        let mut v1 = v1v2_header(1, 2, 2);
+        v1.extend_from_slice(&0x1000u64.to_le_bytes());
+        v1.extend_from_slice(&0u32.to_le_bytes());
+        v1.extend_from_slice(&8u32.to_le_bytes());
+        assert!(matches!(
+            DataLayout::parse(&v1, 8, 8),
+            Err(FormatError::InvalidChunkDimensions(_))
+        ));
+        let mut v4 = vec![4u8, 2, 0, 2, 4];
+        v4.extend_from_slice(&0u32.to_le_bytes());
+        v4.extend_from_slice(&8u32.to_le_bytes());
+        v4.push(3); // fixed array index
+        v4.push(0); // page bits
+        v4.extend_from_slice(&0x1000u64.to_le_bytes());
+        assert!(matches!(
+            DataLayout::parse(&v4, 8, 8),
+            Err(FormatError::InvalidChunkDimensions(_))
+        ));
+    }
+
+    /// A v4 chunked layout (fixed array index) whose `dims` are each
+    /// encoded in `width` bytes.
+    fn v4_chunked_msg(width: u8, dims: &[u64]) -> Vec<u8> {
+        let mut m = vec![4u8, 2, 0, dims.len() as u8, width];
+        for &d in dims {
+            m.extend_from_slice(&d.to_le_bytes()[..width.min(8) as usize]);
+        }
+        m.push(3); // fixed array index
+        m.push(0); // page bits
+        m.extend_from_slice(&0x1000u64.to_le_bytes());
+        m
+    }
+
+    #[test]
+    fn v4_chunk_dimensions_take_1_to_8_bytes() {
+        // libhdf5 encodes each dimension in the fewest bytes that hold the
+        // largest: a chunk dimension of 70 000 takes 3, and 3, 5, 6 and 7
+        // were refused ("UnexpectedEof").
+        for width in 1..=8u8 {
+            let dims = [if width >= 3 { 70_000 } else { 200 }, 8];
+            let layout = DataLayout::parse(&v4_chunked_msg(width, &dims), 8, 8)
+                .unwrap_or_else(|e| panic!("width {width}: {e:?}"));
+            assert!(
+                matches!(&layout, DataLayout::Chunked { chunk_dimensions, .. }
+                    if chunk_dimensions.iter().map(|&d| u64::from(d)).eq(dims)),
+                "width {width}: {layout:?}"
+            );
+        }
+        // libhdf5 refuses 0 and more than 8 bytes.
+        for width in [0u8, 9] {
+            assert_eq!(
+                DataLayout::parse(&v4_chunked_msg(width, &[4, 8]), 8, 8).unwrap_err(),
+                FormatError::InvalidChunkDimensions(
+                    "encoded chunk dimension size is too large".into()
+                )
+            );
+        }
+        // A dimension past u32 cannot be represented and is refused, not
+        // truncated.
+        assert!(matches!(
+            DataLayout::parse(&v4_chunked_msg(5, &[1 << 32, 8]), 8, 8),
+            Err(FormatError::InvalidChunkDimensions(m)) if m.contains("2^32")
+        ));
+    }
+
+    #[test]
+    fn v1v2_rejects_bad_class_dimensionality_and_truncation() {
+        assert_eq!(
+            DataLayout::parse(&v1v2_header(1, 1, 3), 8, 8).unwrap_err(),
+            FormatError::InvalidLayoutClass(3)
+        );
+        assert!(matches!(
+            DataLayout::parse(&v1v2_header(2, 34, 1), 8, 8).unwrap_err(),
+            FormatError::Overflow(_)
+        ));
+        // Chunked, dims cut short.
+        let mut buf = v1v2_header(1, 2, 2);
+        buf.extend_from_slice(&0x10u64.to_le_bytes());
+        buf.extend_from_slice(&7u32.to_le_bytes());
+        assert!(matches!(
+            DataLayout::parse(&buf, 8, 8).unwrap_err(),
+            FormatError::UnexpectedEof { .. }
+        ));
+        // Compact, raw data shorter than its declared size.
+        let mut buf = v1v2_header(2, 1, 0);
+        buf.extend_from_slice(&4u32.to_le_bytes());
+        buf.extend_from_slice(&100u32.to_le_bytes());
+        buf.extend_from_slice(&[0; 4]);
+        assert!(matches!(
+            DataLayout::parse(&buf, 8, 8).unwrap_err(),
+            FormatError::UnexpectedEof { .. }
+        ));
+    }
+
     #[test]
     fn v3_compact() {
         let mut buf = vec![3u8, 0]; // version=3, class=0 (compact)
@@ -849,6 +1202,62 @@ mod tests {
         assert_eq!(v1.iter_linear_1d(8).unwrap(), vec![4, 5, 6, 7]);
     }
 
+    #[test]
+    fn parse_vds_mappings_v1_shared_names() {
+        // Written by HDF5 2.0 (h5py, libver=("v200", "v200")) for three
+        // mappings from `a_rather_long_source_file.h5:a_rather_long_dataset_name`
+        // and one from the same file: the entries carry flags 0x00, 0x03, 0x03
+        // and 0x06, so names after the first are stored as entry indices.
+        let blob: &[u8] = &[
+            0x01, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x61, 0x5f, 0x72, 0x61,
+            0x74, 0x68, 0x65, 0x72, 0x5f, 0x6c, 0x6f, 0x6e, 0x67, 0x5f, 0x73, 0x6f, 0x75, 0x72,
+            0x63, 0x65, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x2e, 0x68, 0x35, 0x00, 0x61, 0x5f, 0x72,
+            0x61, 0x74, 0x68, 0x65, 0x72, 0x5f, 0x6c, 0x6f, 0x6e, 0x67, 0x5f, 0x64, 0x61, 0x74,
+            0x61, 0x73, 0x65, 0x74, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x00, 0x02, 0x00, 0x00, 0x00,
+            0x03, 0x00, 0x00, 0x00, 0x01, 0x02, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00,
+            0x01, 0x00, 0x04, 0x00, 0x02, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x02,
+            0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x00, 0x00,
+            0x01, 0x00, 0x01, 0x00, 0x04, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x03,
+            0x00, 0x00, 0x00, 0x01, 0x02, 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x01, 0x00, 0x01,
+            0x00, 0x04, 0x00, 0x02, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x02, 0x02,
+            0x00, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01,
+            0x00, 0x01, 0x00, 0x04, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x03, 0x00,
+            0x00, 0x00, 0x01, 0x02, 0x01, 0x00, 0x00, 0x00, 0x08, 0x00, 0x01, 0x00, 0x01, 0x00,
+            0x04, 0x00, 0x02, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x02, 0x02, 0x00,
+            0x00, 0x00, 0x02, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00,
+            0x01, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02,
+            0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x02, 0x01, 0x00, 0x00, 0x00, 0x00,
+            0x00, 0x01, 0x00, 0x01, 0x00, 0x04, 0x00, 0x02, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00,
+            0x00, 0x01, 0x02, 0x02, 0x00, 0x00, 0x00, 0x03, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01,
+            0x00, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x04, 0x00, 0x8e, 0xa7, 0xea, 0x7a,
+        ];
+        let mappings = parse_vds_mappings(blob, 8).unwrap();
+        let names: Vec<(&str, &str)> = mappings
+            .iter()
+            .map(|m| (m.source_file.as_str(), m.source_dataset.as_str()))
+            .collect();
+        let (file, dset) = ("a_rather_long_source_file.h5", "a_rather_long_dataset_name");
+        assert_eq!(
+            names,
+            vec![(file, dset), (file, dset), (file, dset), (".", dset)]
+        );
+    }
+
+    #[test]
+    fn parse_vds_mappings_v1_forward_reference_is_error() {
+        // Entry 0 claiming to share entry 0's file name must not index past
+        // the entries decoded so far.
+        let mut blob = vec![0x01u8, 1, 0, 0, 0, 0, 0, 0, 0, 0x01];
+        blob.extend_from_slice(&[0u8; 8]);
+        blob.extend_from_slice(b"d\0");
+        assert!(parse_vds_mappings(&blob, 8).is_err());
+        // Unknown flag bits are refused.
+        let blob = [0x01u8, 1, 0, 0, 0, 0, 0, 0, 0, 0x08, b'd', 0];
+        assert!(parse_vds_mappings(&blob, 8).is_err());
+    }
+
     #[test]
     fn parse_vds_mappings_external_v0() {
         // Block version 0 with an explicit (external) source file name.

crates/clawhdf5-format/src/data_read.rs

@@ -191,14 +191,9 @@ fn read_raw_data_full_impl(
             offset_size,
             length_size,
         ),
-        DataLayout::Virtual {
+        DataLayout::Virtual { .. } => read_virtual_data(
-            global_heap_address,
-            global_heap_index,
-            ..
-        } => read_virtual_data(
             file_data,
-            *global_heap_address,
+            layout,
-            *global_heap_index,
             dataspace,
             datatype,
             offset_size,
@@ -308,6 +303,7 @@ pub fn read_raw_data_selection(
     use crate::selection::Selection;
 
     crate::partial_read::validate(selection, &dataspace.dimensions)?;
+    crate::chunked_read::check_chunk_element_size(layout, datatype, offset_size)?;
 
     // Read only what the selection's bounding box touches when that is
     // possible; everything below is the decode-everything-then-pick path,
@@ -365,6 +361,7 @@ pub fn read_raw_data_selection(
             chunk_index_type,
             ..
         } => {
+            crate::chunked_read::chunk_geometry(chunk_dimensions, *version, dataspace, elem_size)?;
             // For chunked data, only read chunks that intersect the selection
             let chunk_dims: Vec<u64> = chunk_dimensions.iter().map(|&d| d as u64).collect();
             let rank = dims.len();
@@ -405,10 +402,10 @@ pub fn read_raw_data_selection(
             } else {
                 // v3: B-tree v1
                 if let Some(addr) = btree_address {
-                    crate::chunked_read::collect_chunk_info(
+                    crate::chunked_read::collect_chunk_info_checked(
                         file_data,
                         *addr,
-                        rank + 1,
+                        chunk_dimensions,
                         offset_size,
                         length_size,
                     )?
@@ -465,158 +462,54 @@ pub fn read_raw_data_selection(
     }
 }
 
-/// Assemble a **Virtual Dataset (VDS)** from its source mappings.
+/// Assemble a **Virtual Dataset (VDS)** through the raw-read API, which has no
+/// access to the dataset's fill value message.
 ///
-/// Supports virtual datasets of any rank. Same-file sources are read directly;
+/// Delegates to [`crate::vds::read_virtual_dataset`]. Because the fill value
-/// **external-file** sources are read through the caller-supplied `resolver`,
+/// is unknown here, a virtual dataset with any element no mapping supplies
-/// which maps a stored source file name to that file's bytes. Each mapping's
+/// (an unmapped region, or a missing source file or dataset) is an error
-/// selected source elements are scattered into the virtual buffer at the
+/// rather than a guess at the fill value; so is one whose extent libhdf5
-/// positions given by the virtual selection (both enumerated in row-major
+/// would report differently from the stored dataspace (unlimited mappings).
-/// order, as HDF5 pairs them). Unmapped regions are left at the zero fill value.
+/// Use [`crate::vds::read_virtual_dataset`] to read those.
-///
-/// A mapping whose external source file the resolver cannot supply (`None`) is
-/// skipped, leaving its region at fill — matching HDF5's tolerance of missing
-/// sources. An external source with no resolver at all is a hard error.
 #[allow(clippy::too_many_arguments)]
 fn read_virtual_data(
     file_data: &[u8],
-    global_heap_address: Option<u64>,
+    layout: &DataLayout,
-    global_heap_index: u32,
     dataspace: &Dataspace,
     datatype: &Datatype,
     offset_size: u8,
     length_size: u8,
     resolver: Option<&VdsSourceResolver>,
 ) -> Result<Vec<u8>, FormatError> {
-    use crate::data_layout::parse_vds_mappings;
+    let wrapped =
-    use crate::global_heap::GlobalHeapCollection;
+        resolver.map(|r| move |name: &str| -> Result<Option<Vec<u8>>, FormatError> { Ok(r(name)) });
-    use crate::selection::Selection;
+    let wrapped_ref = wrapped.as_ref().map(|w| w as &crate::vds::VdsFileResolver);
-
+    let v = crate::vds::read_virtual_dataset(
-    let elem_size = datatype.type_size() as usize;
-    let mut out = crate::chunked_read::alloc_output(crate::chunked_read::checked_byte_len(
-        dataspace.checked_num_elements()?,
-        elem_size,
-    )?)?;
-
-    let virtual_dims = &dataspace.dimensions;
-
-    let addr = global_heap_address.ok_or_else(|| {
-        FormatError::ChunkedReadError("virtual dataset has no mapping global heap".into())
-    })?;
-    let coll = GlobalHeapCollection::parse(file_data, addr as usize, length_size)?;
-    let obj =
-        coll.get_object(global_heap_index as u16)
-            .ok_or(FormatError::GlobalHeapObjectNotFound {
-                collection_address: addr,
-                index: global_heap_index as u16,
-            })?;
-    let mappings = parse_vds_mappings(&obj.data, length_size)?;
-
-    for m in &mappings {
-        let same_file = m.source_file.is_empty() || m.source_file == ".";
-
-        // Resolve the bytes of the file holding this source dataset.
-        let external;
-        let src_file_data: &[u8] = if same_file {
-            file_data
-        } else {
-            let r = resolver.ok_or_else(|| {
-                FormatError::ChunkedReadError(
-                    "external-file virtual dataset sources require a file resolver".into(),
-                )
-            })?;
-            match r(&m.source_file) {
-                Some(bytes) => {
-                    external = bytes;
-                    &external
-                }
-                // Source file unavailable: leave this region at fill value.
-                None => continue,
-            }
-        };
-
-        let (vsel, _) = Selection::decode_serialized(&m.virtual_selection)?;
-        let (ssel, _) = Selection::decode_serialized(&m.source_selection)?;
-
-        let (src_raw, src_dims) =
-            read_named_dataset_raw(src_file_data, &m.source_dataset, offset_size, length_size)?;
-
-        let vidx = vsel.iter_linear(virtual_dims)?;
-        let sidx = ssel.iter_linear(&src_dims)?;
-        if vidx.len() != sidx.len() {
-            return Err(FormatError::ChunkedReadError(
-                "virtual/source selection element counts differ".into(),
-            ));
-        }
-
-        for (&v, &s) in vidx.iter().zip(sidx.iter()) {
-            let (vo, so) = (v as usize * elem_size, s as usize * elem_size);
-            if vo + elem_size > out.len() || so + elem_size > src_raw.len() {
-                return Err(FormatError::ChunkedReadError(
-                    "virtual dataset selection out of bounds".into(),
-                ));
-            }
-            out[vo..vo + elem_size].copy_from_slice(&src_raw[so..so + elem_size]);
-        }
-    }
-
-    Ok(out)
-}
-
-/// Read a named dataset's raw (decoded) bytes and its dimensions, navigating
-/// from the superblock. Used to pull VDS source datasets out of the same file.
-fn read_named_dataset_raw(
-    file_data: &[u8],
-    path: &str,
-    _offset_size: u8,
-    _length_size: u8,
-) -> Result<(Vec<u8>, Vec<u64>), FormatError> {
-    use crate::filter_pipeline::FilterPipeline;
-    use crate::group_v2::resolve_path_any;
-    use crate::message_type::MessageType;
-    use crate::object_header::ObjectHeader;
-    use crate::signature::find_signature;
-    use crate::superblock::Superblock;
-
-    let sig = find_signature(file_data)?;
-    let sb = Superblock::parse(file_data, sig)?;
-    let addr = resolve_path_any(file_data, &sb, path)?;
-    let hdr = ObjectHeader::parse(file_data, addr as usize, sb.offset_size, sb.length_size)?;
-
-    let find = |t: MessageType| hdr.messages.iter().find(|m| m.msg_type == t);
-    let ds_msg = find(MessageType::Dataspace)
-        .ok_or_else(|| FormatError::ChunkedReadError("VDS source has no dataspace".into()))?;
-    let dataspace = Dataspace::parse(&ds_msg.data, sb.length_size)?;
-    let dt_msg = find(MessageType::Datatype)
-        .ok_or_else(|| FormatError::ChunkedReadError("VDS source has no datatype".into()))?;
-    let (datatype, _) = Datatype::parse(&dt_msg.data)?;
-    let dl_msg = find(MessageType::DataLayout)
-        .ok_or_else(|| FormatError::ChunkedReadError("VDS source has no data layout".into()))?;
-    let layout = DataLayout::parse(&dl_msg.data, sb.offset_size, sb.length_size)?;
-    // A virtual dataset whose source is itself another virtual dataset could
-    // form a cycle (A -> B -> A) and recurse into a stack overflow. Nested
-    // virtual sources are exotic and unsupported, so stop here cleanly.
-    if matches!(layout, DataLayout::Virtual { .. }) {
-        return Err(FormatError::ChunkedReadError(
-            "virtual dataset source is itself virtual (unsupported)".into(),
-        ));
-    }
-    let pipeline = find(MessageType::FilterPipeline)
-        .map(|m| FilterPipeline::parse(&m.data))
-        .transpose()?;
-
-    let raw = read_raw_data_full(
         file_data,
-        &layout,
+        layout,
-        &dataspace,
+        dataspace,
-        &datatype,
+        datatype,
-        pipeline.as_ref(),
+        None,
-        sb.offset_size,
+        offset_size,
-        sb.length_size,
+        length_size,
+        wrapped_ref,
     )?;
-    Ok((raw, dataspace.dimensions.clone()))
+    if v.dims != dataspace.dimensions {
+        return Err(FormatError::ChunkedReadError(
+            "virtual dataset extent differs from its stored dataspace; \
+             read it with vds::read_virtual_dataset"
+                .into(),
+        ));
+    }
+    if v.unmapped > 0 {
+        return Err(FormatError::ChunkedReadError(
+            "virtual dataset has elements no source supplies, which read as its \
+             fill value; read it with vds::read_virtual_dataset and the fill value"
+                .into(),
+        ));
+    }
+    Ok(v.data)
 }
-
 /// Extract selected elements from a full dataset buffer.
 pub fn extract_selection_from_buffer(
     full_data: &[u8],

crates/clawhdf5-format/src/datatype.rs

@@ -208,6 +208,31 @@ fn offset_bytes_for_size(compound_size: u32) -> usize {
 }
 
 /// Read an unsigned integer of 1, 2, 4, or 8 bytes (LE).
+/// The size field of the datatype message at `pos`, as stored (a
+/// variable-length type's stored size is not modelled in [`Datatype`]).
+fn stored_type_size(data: &[u8], pos: usize) -> Result<u32, FormatError> {
+    ensure_len(data, pos, 8)?;
+    Ok(LittleEndian::read_u32(&data[pos + 4..pos + 8]))
+}
+
+/// libhdf5 refuses an array type of more than `H5S_MAX_RANK` (32)
+/// dimensions.
+fn check_array_rank(ndims: usize) -> Result<(), FormatError> {
+    if ndims > 32 {
+        return Err(invalid("too many dimensions for array datatype"));
+    }
+    Ok(())
+}
+
+/// A zero-sized array dimension makes a zero-sized type, which libhdf5
+/// cannot open ("unable to retrieve size of datatype").
+fn check_array_dims(dims: &[u32]) -> Result<(), FormatError> {
+    if dims.contains(&0) {
+        return Err(invalid("zero-sized dimension specified"));
+    }
+    Ok(())
+}
+
 fn read_uint(data: &[u8], offset: usize, nbytes: usize) -> Result<u64, FormatError> {
     ensure_len(data, offset, nbytes)?;
     let slice = &data[offset..offset + nbytes];
@@ -232,10 +257,104 @@ fn read_uint(data: &[u8], offset: usize, nbytes: usize) -> Result<u64, FormatErr
 /// available stack is a few KB.
 const MAX_DATATYPE_DEPTH: u16 = 64;
 
+fn invalid(why: impl Into<String>) -> FormatError {
+    FormatError::InvalidDatatype(why.into())
+}
+
+/// libhdf5's bounds checks on an integer type's bit offset and precision
+/// (`H5O__dtype_decode_helper`): both must lie inside the type. (Newer
+/// libhdf5 checks bit fields the same way; HDF5 2.0, which h5py 3.16 ships,
+/// does not, and opens such a type.)
+fn check_integer_bits(size: u32, bit_offset: u16, bit_precision: u16) -> Result<(), FormatError> {
+    let bits = u64::from(size) * 8;
+    if u64::from(bit_offset) >= bits {
+        return Err(invalid("integer offset out of bounds"));
+    }
+    if bit_precision == 0 {
+        return Err(invalid("precision is zero"));
+    }
+    if u64::from(bit_offset) + u64::from(bit_precision) > bits {
+        return Err(invalid("integer offset+precision out of bounds"));
+    }
+    Ok(())
+}
+
+/// Whether the closed bit ranges `[a0, a1]` and `[b0, b1]` share a bit.
+fn ranges_overlap(a0: u64, a1: u64, b0: u64, b1: u64) -> bool {
+    a0 <= b1 && b0 <= a1
+}
+
+/// libhdf5's checks on a floating-point type's fields: exponent and mantissa
+/// must lie inside the type, be non-empty, and not overlap each other or the
+/// sign bit. (libhdf5 does not check a float's bit offset and precision.)
+///
+/// One libhdf5 check is left out on purpose: a sign bit position outside the
+/// type ("sign bit position out of bounds"). clawhdf5 up to v2.7.0 wrote 63
+/// there for every float, so every `f32` it wrote (every agent store's
+/// embeddings) would stop opening. The position is not used to decode an
+/// IEEE float, so reading such a type returns the right values.
+fn check_float_fields(
+    size: u32,
+    sign: u8,
+    epos: u8,
+    esize: u8,
+    mpos: u8,
+    msize: u8,
+) -> Result<(), FormatError> {
+    let bits = u64::from(size) * 8;
+    let (sign, epos, esize, mpos, msize) = (
+        u64::from(sign),
+        u64::from(epos),
+        u64::from(esize),
+        u64::from(mpos),
+        u64::from(msize),
+    );
+    if esize == 0 {
+        return Err(invalid("exponent size can't be zero"));
+    }
+    if epos >= bits {
+        return Err(invalid("exponent starting position out of bounds"));
+    }
+    if epos + esize > bits {
+        return Err(invalid("exponent range out of bounds"));
+    }
+    if msize == 0 {
+        return Err(invalid("mantissa size can't be zero"));
+    }
+    if mpos >= bits {
+        return Err(invalid("mantissa starting position out of bounds"));
+    }
+    if mpos + msize > bits {
+        return Err(invalid("mantissa range out of bounds"));
+    }
+    let (e_end, m_end) = (epos + esize - 1, mpos + msize - 1);
+    if ranges_overlap(sign, sign, epos, e_end) {
+        return Err(invalid("exponent and sign positions overlap"));
+    }
+    if ranges_overlap(sign, sign, mpos, m_end) {
+        return Err(invalid("mantissa and sign positions overlap"));
+    }
+    if ranges_overlap(epos, e_end, mpos, m_end) {
+        return Err(invalid("mantissa and exponent positions overlap"));
+    }
+    Ok(())
+}
+
 impl Datatype {
     /// Parse a datatype message from raw bytes.
     ///
     /// Returns `(Datatype, bytes_consumed)` for recursive parsing.
+    ///
+    /// A type libhdf5 refuses to decode is refused here too, with
+    /// [`FormatError::InvalidDatatype`] carrying libhdf5's reason: size 0,
+    /// integer/bit-field/float bit fields outside the type or overlapping,
+    /// a compound with no members, a member outside its compound, a
+    /// duplicate or overlapping member, an enum whose size differs from its
+    /// base type's or with an empty name, an array of more than 32
+    /// dimensions or a zero-sized one, an unaligned opaque tag length.
+    /// Reading such a type used to return data from a corrupt file. Checks
+    /// newer libhdf5 releases add but HDF5 2.0 (h5py 3.16) lacks are left
+    /// out, so a file h5py opens still opens here.
     pub fn parse(data: &[u8]) -> Result<(Datatype, usize), FormatError> {
         Self::parse_with_depth(data, 0)
     }
@@ -259,6 +378,14 @@ impl Datatype {
 
         let size = LittleEndian::read_u32(&data[4..8]);
         let mut pos = 8;
+        // libhdf5 refuses size 0 for every class. A fixed-length string is
+        // exempt: clawhdf5 up to v2.7.0 wrote an empty-string attribute
+        // with a size-0 string type, and refusing it would fail every
+        // attribute of such objects, while reading it (an empty string) is
+        // harmless.
+        if size == 0 && class_id != 3 {
+            return Err(invalid("invalid datatype size"));
+        }
 
         match class_id {
             0 => {
@@ -272,6 +399,7 @@ impl Datatype {
                 let signed = (bf0 >> 3) & 0x01 == 1;
                 let bit_offset = LittleEndian::read_u16(&data[pos..pos + 2]);
                 let bit_precision = LittleEndian::read_u16(&data[pos + 2..pos + 4]);
+                check_integer_bits(size, bit_offset, bit_precision)?;
                 pos += 4;
                 Ok((
                     Datatype::FixedPoint {
@@ -289,13 +417,23 @@ impl Datatype {
                 ensure_len(data, pos, 12)?;
                 let bo_low = bf0 & 0x01;
                 let bo_high = (bf0 >> 6) & 0x01;
+                // Bit 6 (with bit 0) is VAX order, defined by version 3; libhdf5
+                // ignores bit 6 in older versions, which this read as VAX,
+                // byte-swapping a little-endian float.
+                let bo_high = if version >= 3 { bo_high } else { 0 };
                 let byte_order = match (bo_high, bo_low) {
                     (0, 0) => DatatypeByteOrder::LittleEndian,
                     (0, 1) => DatatypeByteOrder::BigEndian,
-                    (1, 0) => DatatypeByteOrder::Vax,
+                    (1, 0) => {
+                        return Err(invalid("bad byte order for datatype message"));
+                    }
                     (1, 1) => DatatypeByteOrder::Vax,
                     _ => unreachable!(),
                 };
+                // Bits 4-5: mantissa normalization; 3 is undefined.
+                if (bf0 >> 4) & 0x03 == 3 {
+                    return Err(invalid("unknown floating-point normalization"));
+                }
                 let bit_offset = LittleEndian::read_u16(&data[pos..pos + 2]);
                 let bit_precision = LittleEndian::read_u16(&data[pos + 2..pos + 4]);
                 let exponent_location = data[pos + 4];
@@ -303,6 +441,14 @@ impl Datatype {
                 let mantissa_location = data[pos + 6];
                 let mantissa_size = data[pos + 7];
                 let exponent_bias = LittleEndian::read_u32(&data[pos + 8..pos + 12]);
+                check_float_fields(
+                    size,
+                    bf1,
+                    exponent_location,
+                    exponent_size,
+                    mantissa_location,
+                    mantissa_size,
+                )?;
                 pos += 12;
                 Ok((
                     Datatype::FloatingPoint {
@@ -371,6 +517,10 @@ impl Datatype {
             5 => {
                 // Opaque
                 let tag_len = bf0 as usize;
+                // libhdf5 writes the NUL-padded length, a multiple of 8.
+                if !tag_len.is_multiple_of(8) {
+                    return Err(invalid("opaque flag field must be aligned"));
+                }
                 ensure_len(data, pos, tag_len)?;
                 // The stored tag is NUL-padded to a multiple of 8 bytes; the
                 // tag itself ends at the first NUL (libhdf5 reads it with
@@ -384,7 +534,45 @@ impl Datatype {
             6 => {
                 // Compound
                 let num_members = (bf0 as u16) | ((bf1 as u16) << 8);
-                let mut members = Vec::with_capacity(num_members as usize);
+                if num_members == 0 {
+                    return Err(invalid("invalid number of members: 0"));
+                }
+                let mut members: Vec<CompoundMember> = Vec::with_capacity(num_members as usize);
+                // Each member's size in the compound as libhdf5 decodes it:
+                // its stored size, times a v1 member's array dimensions. A
+                // variable-length member takes 4 + offset size + 4 bytes on
+                // disk, not the 16 of `Datatype::type_size`.
+                let mut member_sizes: Vec<u64> = Vec::with_capacity(num_members as usize);
+                // libhdf5 checks each member as it is decoded: it must fit in
+                // the compound (by its own stored size, before a v1 member's
+                // array dimensions are applied), and must not repeat a name
+                // or overlap an earlier member (by its final size).
+                let check_member = |members: &[CompoundMember],
+                                    member_sizes: &[u64],
+                                    name: &str,
+                                    byte_offset: u64,
+                                    stored_size: u32,
+                                    final_size: u64|
+                 -> Result<(), FormatError> {
+                    if byte_offset + u64::from(stored_size) > u64::from(size) {
+                        return Err(invalid(
+                            "member type extends outside its parent compound type",
+                        ));
+                    }
+                    if let Some(j) = members.iter().position(|m| m.name == name) {
+                        return Err(invalid(format!(
+                            "duplicated compound field name '{name}', for fields {j} and {}",
+                            members.len()
+                        )));
+                    }
+                    let end = byte_offset + final_size;
+                    if members.iter().zip(member_sizes).any(|(m, &m_size)| {
+                        byte_offset < m.byte_offset + m_size && m.byte_offset < end
+                    }) {
+                        return Err(invalid("member overlaps with previous member"));
+                    }
+                    Ok(())
+                };
 
                 if (3..=5).contains(&version) {
                     // v3, v4 and v5 share the compact member encoding (name,
@@ -396,9 +584,20 @@ impl Datatype {
                         pos += name_len;
                         let byte_offset = read_uint(data, pos, ob)?;
                         pos += ob;
+                        let stored_size = stored_type_size(data, pos)?;
                         let (member_dt, consumed) =
                             Self::parse_with_depth(&data[pos..], depth + 1)?;
                         pos += consumed;
+                        let final_size = u64::from(stored_size);
+                        check_member(
+                            &members,
+                            &member_sizes,
+                            &name,
+                            byte_offset,
+                            stored_size,
+                            final_size,
+                        )?;
+                        member_sizes.push(final_size);
                         members.push(CompoundMember {
                             name,
                             byte_offset,
@@ -423,13 +622,57 @@ impl Datatype {
                         ensure_len(data, pos, 4)?;
                         let byte_offset = LittleEndian::read_u32(&data[pos..pos + 4]) as u64;
                         pos += 4;
+                        // v1 members can be fixed-size arrays of the member
+                        // type (libhdf5 builds an array type from these
+                        // fields; the permutation is ignored, as libhdf5
+                        // does). Skipping them read a `[4] i32` member as
+                        // one `i32`.
+                        let mut array_dims = Vec::new();
                         if version == 1 {
                             ensure_len(data, pos, 28)?;
+                            let ndims = data[pos] as usize;
+                            // libhdf5 refuses more than four dimensions and,
+                            // when building the array type, a zero-sized one.
+                            let zero_dim = (0..ndims.min(4)).any(|j| {
+                                let at = pos + 12 + 4 * j;
+                                LittleEndian::read_u32(&data[at..at + 4]) == 0
+                            });
+                            if ndims > 4 {
+                                return Err(invalid("invalid number of dimensions for array"));
+                            }
+                            if zero_dim {
+                                return Err(invalid("zero-sized dimension specified"));
+                            }
+                            array_dims = (0..ndims)
+                                .map(|j| {
+                                    let at = pos + 12 + 4 * j;
+                                    LittleEndian::read_u32(&data[at..at + 4])
+                                })
+                                .collect();
                             pos += 28;
                         }
-                        let (member_dt, consumed) =
+                        let stored_size = stored_type_size(data, pos)?;
+                        let (mut member_dt, consumed) =
                             Self::parse_with_depth(&data[pos..], depth + 1)?;
                         pos += consumed;
+                        let final_size = array_dims.iter().fold(u64::from(stored_size), |a, &d| {
+                            a.saturating_mul(u64::from(d))
+                        });
+                        if !array_dims.is_empty() {
+                            member_dt = Datatype::Array {
+                                base_type: Box::new(member_dt),
+                                dimensions: array_dims,
+                            };
+                        }
+                        check_member(
+                            &members,
+                            &member_sizes,
+                            &name,
+                            byte_offset,
+                            stored_size,
+                            final_size,
+                        )?;
+                        member_sizes.push(final_size);
                         members.push(CompoundMember {
                             name,
                             byte_offset,
@@ -468,6 +711,9 @@ impl Datatype {
                 let (base_type, base_consumed) = Self::parse_with_depth(&data[pos..], depth + 1)?;
                 pos += base_consumed;
                 let base_size = base_type.type_size();
+                if base_size != size {
+                    return Err(invalid("ENUM datatype size does not match parent"));
+                }
                 let mut members = Vec::with_capacity(num_members as usize);
                 // Enum layout: base_type, then all names (null-terminated), then all values
                 // v1/v2: names are padded to 8-byte boundaries
@@ -475,6 +721,9 @@ impl Datatype {
                 let mut member_names = Vec::with_capacity(num_members as usize);
                 for _ in 0..num_members {
                     let (name, name_len) = read_null_terminated_string(data, pos)?;
+                    if name.is_empty() {
+                        return Err(invalid("0 length enum name"));
+                    }
                     if version < 3 {
                         let padded = (name_len + 7) & !7;
                         pos += padded;
@@ -535,6 +784,7 @@ impl Datatype {
                 if version == 2 {
                     ensure_len(data, pos, 4)?;
                     let ndims = data[pos] as usize;
+                    check_array_rank(ndims)?;
                     pos += 4; // ndims(1) + reserved(3)
                     ensure_len(data, pos, ndims * 4 + ndims * 4)?;
                     let mut dimensions = Vec::with_capacity(ndims);
@@ -542,6 +792,7 @@ impl Datatype {
                         dimensions.push(LittleEndian::read_u32(&data[pos..pos + 4]));
                         pos += 4;
                     }
+                    check_array_dims(&dimensions)?;
                     // skip permutation indices
                     pos += ndims * 4;
                     let (base_type, consumed) = Self::parse_with_depth(&data[pos..], depth + 1)?;
@@ -558,6 +809,7 @@ impl Datatype {
                     // type); HDF5 1.14+/2.0 with `libver=latest` emits v5.
                     ensure_len(data, pos, 1)?;
                     let ndims = data[pos] as usize;
+                    check_array_rank(ndims)?;
                     pos += 1;
                     ensure_len(data, pos, ndims * 4)?;
                     let mut dimensions = Vec::with_capacity(ndims);
@@ -565,6 +817,7 @@ impl Datatype {
                         dimensions.push(LittleEndian::read_u32(&data[pos..pos + 4]));
                         pos += 4;
                     }
+                    check_array_dims(&dimensions)?;
                     let (base_type, consumed) = Self::parse_with_depth(&data[pos..], depth + 1)?;
                     pos += consumed;
                     Ok((
@@ -621,6 +874,70 @@ impl Datatype {
         }
     }
 
+    /// [`Self::parse`] for the datatype message of an object whose header
+    /// has version `header_version`: a version-1 header, which has no
+    /// checksum, additionally gets [`Self::check_unused_bits`], as libhdf5
+    /// does. Use this wherever the header is at hand.
+    pub fn parse_in_header(
+        data: &[u8],
+        header_version: u8,
+    ) -> Result<(Datatype, usize), FormatError> {
+        let parsed = Self::parse(data)?;
+        if header_version == 1 {
+            parsed.0.check_unused_bits()?;
+        }
+        Ok(parsed)
+    }
+
+    /// libhdf5's guard against a corrupt numeric type in a header without
+    /// a checksum (`H5T_is_numeric_with_unusual_unused_bits`, HDF5 1.14.4+):
+    /// an integer, float or bit field wider than a byte whose precision and
+    /// offset leave more than half its bits unused is taken for corruption
+    /// (e.g. a 3-bit integer in 4 bytes, `cve-2024-29162`, or a 32-bit float
+    /// in 65525 bytes, `cve-2024-32614`), anywhere in the type. libhdf5
+    /// skips the check for checksummed (version-2) headers and when the
+    /// file is opened with `H5Pset_relax_file_integrity_checks`; so does
+    /// [`Self::parse_in_header`], which has no such option.
+    pub fn check_unused_bits(&self) -> Result<(), FormatError> {
+        match self {
+            Datatype::FixedPoint {
+                size,
+                bit_offset,
+                bit_precision,
+                ..
+            }
+            | Datatype::FloatingPoint {
+                size,
+                bit_offset,
+                bit_precision,
+                ..
+            }
+            | Datatype::BitField {
+                size,
+                bit_offset,
+                bit_precision,
+                ..
+            } => {
+                let bits = u64::from(*size) * 8;
+                let prec = u64::from(*bit_precision);
+                if *size > 1 && prec < bits && bits > 2 * (prec + u64::from(*bit_offset)) {
+                    return Err(invalid(format!(
+                        "datatype has unusually large # of unused bits (prec = {prec} bits, \
+                         size = {size} bytes), possibly corrupted file"
+                    )));
+                }
+                Ok(())
+            }
+            Datatype::Compound { members, .. } => members
+                .iter()
+                .try_for_each(|m| m.datatype.check_unused_bits()),
+            Datatype::Enumeration { base_type, .. }
+            | Datatype::VariableLength { base_type, .. }
+            | Datatype::Array { base_type, .. } => base_type.check_unused_bits(),
+            _ => Ok(()),
+        }
+    }
+
     /// Serialize datatype to HDF5 message bytes.
     pub fn serialize(&self) -> Vec<u8> {
         match self {
@@ -832,9 +1149,23 @@ impl Datatype {
     }
 
     /// Check that this datatype can be written: every part of it has an
-    /// on-disk encoding. [`Self::serialize`] cannot report errors, so the
+    /// on-disk encoding, and the encoding is one the reader (and libhdf5)
-    /// writer calls this first.
+    /// accepts. [`Self::serialize`] cannot report errors, so the writer calls
+    /// this first. A compound with no fields or a repeated field name, or an
+    /// enum member with an empty name, is refused here: libhdf5 and h5py
+    /// refuse such types, and so does [`Self::parse`], so writing one made a
+    /// file that could not be read back.
     pub fn check_encodable(&self) -> Result<(), FormatError> {
+        self.check_encodable_parts()?;
+        Self::parse(&self.serialize()).map_err(|e| {
+            FormatError::SerializationError(format!(
+                "datatype cannot be written: HDF5 readers refuse it ({e})"
+            ))
+        })?;
+        Ok(())
+    }
+
+    fn check_encodable_parts(&self) -> Result<(), FormatError> {
         match self {
             Datatype::Opaque { tag, .. } if opaque_tag_text(tag).len() > MAX_OPAQUE_TAG_LEN => {
                 Err(FormatError::SerializationError(format!(
@@ -847,10 +1178,10 @@ impl Datatype {
             )),
             Datatype::Compound { members, .. } => members
                 .iter()
-                .try_for_each(|m| m.datatype.check_encodable()),
+                .try_for_each(|m| m.datatype.check_encodable_parts()),
             Datatype::Enumeration { base_type, .. }
             | Datatype::VariableLength { base_type, .. }
-            | Datatype::Array { base_type, .. } => base_type.check_encodable(),
+            | Datatype::Array { base_type, .. } => base_type.check_encodable_parts(),
             _ => Ok(()),
         }
     }
@@ -954,7 +1285,8 @@ mod tests {
     ) -> Vec<u8> {
         // LE byte order: bo_low=0, bo_high=0
         let bf0 = 0x00u8;
-        let bf1 = 0x00u8;
+        // Sign bit: the top bit.
+        let bf1 = (size * 8 - 1) as u8;
         // mantissa norm = 2 (MSB not stored) in bits 24-31... wait, that's bf2
         let bf2 = 0x02u8; // norm = 2
         let mut buf = build_dt_header(1, 1, [bf0, bf1, bf2], size);
@@ -981,7 +1313,7 @@ mod tests {
         let levels = MAX_DATATYPE_DEPTH as usize + 10;
         let mut data = Vec::new();
         for _ in 0..levels {
-            data.extend_from_slice(&build_dt_header(9, 3, [0, 0, 0], 0));
+            data.extend_from_slice(&build_dt_header(9, 3, [0, 0, 0], 16));
         }
         data.extend_from_slice(&build_fixed_point(4, false, false, 0, 32));
 
@@ -995,7 +1327,7 @@ mod tests {
         let levels = MAX_DATATYPE_DEPTH as usize - 1;
         let mut data = Vec::new();
         for _ in 0..levels {
-            data.extend_from_slice(&build_dt_header(9, 3, [0, 0, 0], 0));
+            data.extend_from_slice(&build_dt_header(9, 3, [0, 0, 0], 16));
         }
         data.extend_from_slice(&build_fixed_point(4, false, false, 0, 32));
 
@@ -1145,8 +1477,8 @@ mod tests {
 
     #[test]
     fn test_opaque() {
-        // tag_len = 4, tag = "BLOB"
+        // tag = "BLOB"; the stored length is the NUL-padded length, 8
-        let mut buf = build_dt_header(5, 1, [4, 0, 0], 64);
+        let mut buf = build_dt_header(5, 1, [8, 0, 0], 64);
         buf.extend_from_slice(b"BLOB");
         // Pad to 8 bytes
         buf.extend_from_slice(&[0, 0, 0, 0]);
@@ -1336,6 +1668,64 @@ mod tests {
         assert_xyid_compound(dt);
     }
 
+    #[test]
+    fn test_compound_v1_member_array_fields() {
+        // HDF5 1.6 wrote array members of a v1 compound through the legacy
+        // per-member fields (as in libhdf5's tools/test/testfiles/
+        // tcompound.h5 `type2`: `int_array` [4] i32, `float_array` [5][6]
+        // f32). They used to be skipped, reading each member as a scalar.
+        let i32le: [u8; 12] = [
+            0x10, 0x08, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x00,
+        ];
+        let mut b = vec![0x16, 0x02, 0x00, 0x00, 0x88, 0x00, 0x00, 0x00];
+        for (name, offset, dims) in [
+            (&b"int_array"[..], 0u32, &[4u32][..]),
+            (&b"xy"[..], 16, &[5u32, 6][..]),
+        ] {
+            let mut padded = name.to_vec();
+            padded.resize((name.len() + 1 + 7) & !7, 0);
+            b.extend_from_slice(&padded);
+            b.extend_from_slice(&offset.to_le_bytes());
+            b.push(dims.len() as u8);
+            b.extend_from_slice(&[0u8; 3 + 4 + 4]); // reserved, permutation, reserved
+            for j in 0..4 {
+                b.extend_from_slice(&dims.get(j).copied().unwrap_or(0).to_le_bytes());
+            }
+            b.extend_from_slice(&i32le);
+        }
+        let (dt, consumed) = Datatype::parse(&b).unwrap();
+        assert_eq!(consumed, b.len());
+        let Datatype::Compound { members, .. } = dt else {
+            panic!("expected Compound, got {dt:?}");
+        };
+        let got: Vec<(&str, u64, u32, Option<Vec<u32>>)> = members
+            .iter()
+            .map(|m| {
+                let dims = match &m.datatype {
+                    Datatype::Array { dimensions, .. } => Some(dimensions.clone()),
+                    _ => None,
+                };
+                (m.name.as_str(), m.byte_offset, m.datatype.type_size(), dims)
+            })
+            .collect();
+        assert_eq!(
+            got,
+            vec![
+                ("int_array", 0, 16, Some(vec![4])),
+                ("xy", 16, 120, Some(vec![5, 6])),
+            ]
+        );
+
+        // More than four dimensions cannot be encoded, and libhdf5 refuses a
+        // zero-sized dimension (a fuzzed tcompound.h5, cve-2024-32616.h5).
+        let mut bad = b.clone();
+        bad[8 + 16 + 4] = 5;
+        assert!(Datatype::parse(&bad).is_err());
+        let mut bad = b.clone();
+        bad[8 + 16 + 4] = 2; // [4, 0]
+        assert!(Datatype::parse(&bad).is_err());
+    }
+
     #[test]
     fn test_compound_v1_truncated_is_error_not_panic() {
         let bytes = compound_v1_bytes();
@@ -1960,4 +2350,273 @@ mod tests {
         };
         assert_eq!(dt.type_size(), 48);
     }
+    /// Every check here mirrors one in libhdf5's `H5O__dtype_decode_helper`;
+    /// the error text is libhdf5's.
+    fn invalid_reason(data: &[u8]) -> String {
+        match Datatype::parse(data) {
+            Err(FormatError::InvalidDatatype(why)) => why,
+            other => panic!("expected InvalidDatatype, got {other:?}"),
+        }
+    }
+
+    #[test]
+    fn size_zero_is_refused() {
+        // cve-2017-17508: a variable-length string member of stored size 0.
+        let mut data = build_dt_header(9, 1, [1, 0, 0], 0);
+        data.extend_from_slice(&build_fixed_point(1, false, false, 0, 8));
+        assert_eq!(invalid_reason(&data), "invalid datatype size");
+        // Except a fixed-length string, which clawhdf5 <= v2.7.0 wrote for an
+        // empty-string attribute.
+        assert!(Datatype::parse(&build_dt_header(3, 1, [0, 0, 0], 0)).is_ok());
+        assert_eq!(
+            invalid_reason(&build_fixed_point(0, false, false, 0, 0)),
+            "invalid datatype size"
+        );
+    }
+
+    #[test]
+    fn integer_bits_must_lie_inside_the_type() {
+        assert_eq!(
+            invalid_reason(&build_fixed_point(4, false, false, 32, 1)),
+            "integer offset out of bounds"
+        );
+        assert_eq!(
+            invalid_reason(&build_fixed_point(4, false, false, 0, 0)),
+            "precision is zero"
+        );
+        assert_eq!(
+            invalid_reason(&build_fixed_point(4, false, false, 8, 25)),
+            "integer offset+precision out of bounds"
+        );
+        // A partial-precision integer inside its bytes is fine.
+        assert!(Datatype::parse(&build_fixed_point(4, false, false, 12, 8)).is_ok());
+    }
+
+    #[test]
+    fn float_fields_must_lie_inside_the_type_and_not_overlap() {
+        // (sign, epos, esize, mpos, msize) on an f32
+        let f32_with = |sign: u8, epos: u8, esize: u8, mpos: u8, msize: u8| {
+            let mut data = build_dt_header(1, 1, [0x20, sign, 0], 4);
+            data.extend_from_slice(&0u16.to_le_bytes());
+            data.extend_from_slice(&32u16.to_le_bytes());
+            data.extend_from_slice(&[epos, esize, mpos, msize]);
+            data.extend_from_slice(&127u32.to_le_bytes());
+            data
+        };
+        assert!(Datatype::parse(&f32_with(31, 23, 8, 0, 23)).is_ok());
+        for (fields, why) in [
+            ((31, 23, 0, 0, 23), "exponent size can't be zero"),
+            (
+                (31, 32, 8, 0, 23),
+                "exponent starting position out of bounds",
+            ),
+            ((31, 30, 8, 0, 23), "exponent range out of bounds"),
+            ((31, 23, 8, 0, 0), "mantissa size can't be zero"),
+            (
+                (31, 23, 8, 40, 1),
+                "mantissa starting position out of bounds",
+            ),
+            // cve-2024-29163: a 128-bit mantissa in a 4-byte float.
+            ((31, 23, 8, 0, 128), "mantissa range out of bounds"),
+            ((23, 23, 8, 0, 23), "exponent and sign positions overlap"),
+            ((0, 23, 8, 0, 23), "mantissa and sign positions overlap"),
+            // cve-2026-34734.
+            (
+                (31, 20, 8, 0, 23),
+                "mantissa and exponent positions overlap",
+            ),
+        ] {
+            let (sign, epos, esize, mpos, msize) = fields;
+            assert_eq!(
+                invalid_reason(&f32_with(sign, epos, esize, mpos, msize)),
+                why,
+                "{fields:?}"
+            );
+        }
+        // Normalization 3 is undefined; bit 6 (VAX) needs bit 0 from v3.
+        let mut data = f32_with(31, 23, 8, 0, 23);
+        data[1] = 0x30;
+        assert_eq!(
+            invalid_reason(&data),
+            "unknown floating-point normalization"
+        );
+        let mut data = f32_with(31, 23, 8, 0, 23);
+        data[0] = 0x31; // version 3
+        data[1] = 0x60;
+        assert_eq!(invalid_reason(&data), "bad byte order for datatype message");
+    }
+
+    #[test]
+    fn unusual_unused_bits_are_refused_in_version_1_headers_only() {
+        // cve-2024-29162: a 3-bit integer in 4 bytes.
+        let data = build_fixed_point(4, false, true, 0, 3);
+        assert!(Datatype::parse_in_header(&data, 2).is_ok());
+        assert_eq!(
+            match Datatype::parse_in_header(&data, 1) {
+                Err(FormatError::InvalidDatatype(why)) => why,
+                other => panic!("{other:?}"),
+            },
+            "datatype has unusually large # of unused bits (prec = 3 bits, size = 4 bytes), \
+             possibly corrupted file"
+        );
+        // Half the bits used (with the offset) is not unusual; nor is a
+        // 1-byte type; nor a full-precision one.
+        for (size, offset, prec) in [(4u32, 0u16, 16u16), (4, 8, 8), (1, 0, 1), (8, 0, 64)] {
+            let data = build_fixed_point(size, false, true, offset, prec);
+            assert!(
+                Datatype::parse_in_header(&data, 1).is_ok(),
+                "{size} {offset} {prec}"
+            );
+        }
+        // Nested: a compound member's type is checked too.
+        let member = build_fixed_point(4, false, true, 0, 15);
+        let data = compound_v3(4, &[("a", 0, member)]);
+        assert!(Datatype::parse_in_header(&data, 2).is_ok());
+        assert!(Datatype::parse_in_header(&data, 1).is_err());
+    }
+
+    #[test]
+    fn f32_written_by_clawhdf5_up_to_2_7_0_still_parses() {
+        // Those versions put the sign bit at 63 whatever the float's size;
+        // libhdf5 refuses it ("sign bit position out of bounds").
+        let mut data = build_dt_header(1, 1, [0x20, 63, 0], 4);
+        data.extend_from_slice(&0u16.to_le_bytes());
+        data.extend_from_slice(&32u16.to_le_bytes());
+        data.extend_from_slice(&[23, 8, 0, 23]);
+        data.extend_from_slice(&127u32.to_le_bytes());
+        assert!(Datatype::parse(&data).is_ok());
+    }
+
+    #[test]
+    fn float_bit_6_is_vax_order_only_from_version_3() {
+        // h5py opens a v1 float with bit 6 set as an ordinary little-endian
+        // float; it used to be read as VAX order.
+        let mut data = build_float(4, 23, 8, 0, 23, 127);
+        data[1] |= 0x40;
+        match Datatype::parse(&data).unwrap().0 {
+            Datatype::FloatingPoint { byte_order, .. } => {
+                assert_eq!(byte_order, DatatypeByteOrder::LittleEndian)
+            }
+            other => panic!("{other:?}"),
+        }
+        data[0] = 0x31;
+        data[1] |= 0x01;
+        match Datatype::parse(&data).unwrap().0 {
+            Datatype::FloatingPoint { byte_order, .. } => {
+                assert_eq!(byte_order, DatatypeByteOrder::Vax)
+            }
+            other => panic!("{other:?}"),
+        }
+    }
+
+    #[test]
+    fn opaque_tag_length_must_be_padded() {
+        let mut data = build_dt_header(5, 1, [4, 0, 0], 4);
+        data.extend_from_slice(b"BLOB");
+        assert_eq!(invalid_reason(&data), "opaque flag field must be aligned");
+    }
+
+    /// A v3 compound of `size` bytes with `(name, offset, member)` members.
+    fn compound_v3(size: u32, members: &[(&str, u8, Vec<u8>)]) -> Vec<u8> {
+        let n = members.len() as u8;
+        let mut data = build_dt_header(6, 3, [n, 0, 0], size);
+        for (name, off, dt) in members {
+            data.extend_from_slice(name.as_bytes());
+            data.push(0);
+            data.push(*off);
+            data.extend_from_slice(dt);
+        }
+        data
+    }
+
+    #[test]
+    fn compound_members_are_checked() {
+        let i4 = build_fixed_point(4, false, true, 0, 32);
+        // cve-2016-4332: no members.
+        assert_eq!(
+            invalid_reason(&compound_v3(8, &[])),
+            "invalid number of members: 0"
+        );
+        assert_eq!(
+            invalid_reason(&compound_v3(
+                8,
+                &[("a", 0, i4.clone()), ("b", 6, i4.clone())]
+            )),
+            "member type extends outside its parent compound type"
+        );
+        assert_eq!(
+            invalid_reason(&compound_v3(
+                8,
+                &[("a", 0, i4.clone()), ("a", 4, i4.clone())]
+            )),
+            "duplicated compound field name 'a', for fields 0 and 1"
+        );
+        assert_eq!(
+            invalid_reason(&compound_v3(
+                8,
+                &[("a", 0, i4.clone()), ("b", 2, i4.clone())]
+            )),
+            "member overlaps with previous member"
+        );
+        assert_eq!(
+            invalid_reason(&compound_v3(
+                8,
+                &[("b", 4, i4.clone()), ("a", 2, i4.clone())]
+            )),
+            "member overlaps with previous member"
+        );
+        // Members out of offset order, and gaps, are fine.
+        assert!(Datatype::parse(&compound_v3(12, &[("b", 8, i4.clone()), ("a", 0, i4)])).is_ok());
+    }
+
+    #[test]
+    fn enum_is_checked() {
+        let base = build_fixed_point(4, false, true, 0, 32);
+        let enum_of = |size: u32, names: &[&str]| {
+            let mut data = build_dt_header(8, 3, [names.len() as u8, 0, 0], size);
+            data.extend_from_slice(&base);
+            for n in names {
+                data.extend_from_slice(n.as_bytes());
+                data.push(0);
+            }
+            for i in 0..names.len() as u32 {
+                data.extend_from_slice(&i.to_le_bytes());
+            }
+            data
+        };
+        assert!(Datatype::parse(&enum_of(4, &["RED", "GREEN"])).is_ok());
+        // cve-2024-32618.
+        assert_eq!(
+            invalid_reason(&enum_of(4, &["", "GREEN"])),
+            "0 length enum name"
+        );
+        assert_eq!(
+            invalid_reason(&enum_of(2, &["RED"])),
+            "ENUM datatype size does not match parent"
+        );
+    }
+
+    #[test]
+    fn array_dimensions_are_checked() {
+        let base = build_fixed_point(4, false, true, 0, 32);
+        let array_v3 = |dims: &[u32]| {
+            let n = dims.iter().product::<u32>().max(1);
+            let mut data = build_dt_header(10, 3, [0, 0, 0], 4 * n);
+            data.push(dims.len() as u8);
+            for d in dims {
+                data.extend_from_slice(&d.to_le_bytes());
+            }
+            data.extend_from_slice(&base);
+            data
+        };
+        assert!(Datatype::parse(&array_v3(&[2, 3])).is_ok());
+        assert_eq!(
+            invalid_reason(&array_v3(&[2, 0])),
+            "zero-sized dimension specified"
+        );
+        assert_eq!(
+            invalid_reason(&array_v3(&[1; 33])),
+            "too many dimensions for array datatype"
+        );
+    }
 }

crates/clawhdf5-format/src/ea_writer.rs

@@ -7,7 +7,9 @@ extern crate alloc;
 use alloc::{vec, vec::Vec};
 
 use crate::checksum::jenkins_lookup3;
-use crate::chunked_write::{WrittenChunk, filtered_chunk_size_len, push_addr, push_index_element};
+use crate::chunked_write::{
+    WrittenChunk, filtered_chunk_size_len, push_addr, push_index_element, push_v4_chunk_dims,
+};
 
 /// Serialize a v4 Extensible Array layout message.
 pub(crate) fn serialize_v4_extensible_array(
@@ -24,35 +26,7 @@ pub(crate) fn serialize_v4_extensible_array(
     let ndims = chunk_dims.len() as u8 + 1;
     buf.push(ndims);
 
-    let max_dim = chunk_dims
+    push_v4_chunk_dims(&mut buf, chunk_dims, element_size);
-        .iter()
-        .map(|&d| d as u64)
-        .chain(core::iter::once(element_size as u64))
-        .max()
-        .unwrap_or(1);
-    let dim_encoded_len: u8 = if max_dim <= 0xFF {
-        1
-    } else if max_dim <= 0xFFFF {
-        2
-    } else {
-        4
-    };
-    buf.push(dim_encoded_len);
-
-    for &d in chunk_dims {
-        match dim_encoded_len {
-            1 => buf.push(d as u8),
-            2 => buf.extend_from_slice(&(d as u16).to_le_bytes()),
-            4 => buf.extend_from_slice(&d.to_le_bytes()),
-            _ => unreachable!("unexpected dim_encoded_len: {dim_encoded_len}"),
-        }
-    }
-    match dim_encoded_len {
-        1 => buf.push(element_size as u8),
-        2 => buf.extend_from_slice(&(element_size as u16).to_le_bytes()),
-        4 => buf.extend_from_slice(&element_size.to_le_bytes()),
-        _ => unreachable!("unexpected dim_encoded_len: {dim_encoded_len}"),
-    }
 
     // chunk index type = 4 (Extensible Array)
     buf.push(4);

crates/clawhdf5-format/src/error.rs

@@ -80,6 +80,9 @@ pub enum FormatError {
     InvalidLocalHeapSignature,
     /// Invalid local heap version.
     InvalidLocalHeapVersion(u8),
+    /// A local heap's free list points outside its data segment (libhdf5:
+    /// "bad heap free list").
+    InvalidLocalHeapFreeList,
     /// Invalid B-tree v1 signature.
     InvalidBTreeSignature,
     /// Invalid B-tree node type.
@@ -117,6 +120,14 @@ pub enum FormatError {
     /// A message is marked shared but was parsed without access to the file,
     /// so the reference to the real message could not be followed.
     UnresolvedSharedMessage,
+    /// A shared-message reference points at an object header that holds no
+    /// (unshared) message of the referenced type (raw message type id).
+    SharedMessageTargetMissing(u16),
+    /// A superblock was parsed at a non-zero offset of the buffer (the file
+    /// has a user block of this many bytes). HDF5 addresses are relative to
+    /// the superblock, so the buffer must start there: see
+    /// `signature::split_user_block`.
+    UserBlockNotStripped(u64),
     /// A selection does not fit the dataset it was applied to (wrong rank, or
     /// it reaches past a dimension's extent).
     SelectionOutOfBounds(String),
@@ -190,6 +201,28 @@ pub enum FormatError {
     DuplicateDatasetName(String),
     /// Integer overflow in size computation (malformed data protection).
     Overflow(String),
+    /// An object header that libhdf5 refuses to load (the reason is
+    /// libhdf5's own error text): a misaligned or overrunning message, a
+    /// wrong message count, contradictory message flags, a message of a
+    /// class that cannot be shared flagged shareable, …
+    InvalidObjectHeader(&'static str),
+    /// A datatype message libhdf5 refuses to decode (the reason is
+    /// libhdf5's own error text): size 0, bit fields outside the type,
+    /// an empty enum name, a compound member outside its compound, …
+    InvalidDatatype(String),
+    /// A chunked layout whose chunk dimensions libhdf5 refuses: a zero
+    /// dimension, a rank that does not match the dataspace, an element size
+    /// that is not the datatype's, or a chunk of 4 GiB or more indexed by a
+    /// version-1 B-tree.
+    InvalidChunkDimensions(String),
+    /// The superblock's end-of-file address lies past the end of the file:
+    /// the file was truncated (libhdf5 refuses to open it).
+    TruncatedFile {
+        /// End of file recorded in the superblock (relative to byte 0).
+        stored_eof: u64,
+        /// The file's actual length in bytes.
+        actual_len: u64,
+    },
 }
 
 impl fmt::Display for FormatError {
@@ -270,6 +303,9 @@ impl fmt::Display for FormatError {
             FormatError::InvalidLocalHeapSignature => {
                 write!(f, "invalid local heap signature")
             }
+            FormatError::InvalidLocalHeapFreeList => {
+                write!(f, "bad local heap free list")
+            }
             FormatError::InvalidLocalHeapVersion(v) => {
                 write!(f, "invalid local heap version: {v}")
             }
@@ -339,6 +375,16 @@ impl fmt::Display for FormatError {
             FormatError::SelectionOutOfBounds(msg) => {
                 write!(f, "selection out of bounds: {msg}")
             }
+            FormatError::UserBlockNotStripped(n) => write!(
+                f,
+                "file has a {n}-byte user block: parse the bytes from the superblock on \
+                 (signature::split_user_block)"
+            ),
+            FormatError::SharedMessageTargetMissing(t) => write!(
+                f,
+                "shared message reference points at an object header with no message of type \
+                 {t:#06x}"
+            ),
             FormatError::UnresolvedSharedMessage => write!(
                 f,
                 "message is shared but no file data was available to resolve it"
@@ -382,9 +428,17 @@ impl fmt::Display for FormatError {
             FormatError::InvalidFilterPipelineVersion(v) => {
                 write!(f, "invalid filter pipeline version: {v}")
             }
-            FormatError::UnsupportedFilter(id) => {
+            FormatError::UnsupportedFilter(id) => match crate::filter_registry::known_filter(*id) {
-                write!(f, "unsupported filter: {id}")
+                Some((name, Some(feature))) => write!(
-            }
+                    f,
+                    "unsupported filter: {id} ({name}; this build lacks the `{feature}` feature)"
+                ),
+                Some((name, None)) => write!(
+                    f,
+                    "unsupported filter: {id} ({name}, not implemented by clawhdf5)"
+                ),
+                None => write!(f, "unsupported filter: {id}"),
+            },
             FormatError::FilterError(msg) => {
                 write!(f, "filter error: {msg}")
             }
@@ -421,6 +475,25 @@ impl fmt::Display for FormatError {
             FormatError::Overflow(msg) => {
                 write!(f, "integer overflow: {msg}")
             }
+            FormatError::InvalidObjectHeader(why) => {
+                write!(f, "corrupt object header: {why}")
+            }
+            FormatError::InvalidDatatype(why) => {
+                write!(f, "invalid datatype: {why}")
+            }
+            FormatError::InvalidChunkDimensions(why) => {
+                write!(f, "invalid chunk dimensions: {why}")
+            }
+            FormatError::TruncatedFile {
+                stored_eof,
+                actual_len,
+            } => {
+                write!(
+                    f,
+                    "truncated file: the superblock records end of file {stored_eof}, \
+                     but the file is {actual_len} bytes"
+                )
+            }
         }
     }
 }

crates/clawhdf5-format/src/file_writer.rs

@@ -1947,6 +1947,12 @@ mod tests {
             root_block_address: 0,
             current_rows_in_root_indirect_block: 0,
             managed_objects_count: 0,
+            huge_btree_address: u64::MAX,
+            filter_pipeline: None,
+            root_direct_block_filtered_size: 0,
+            root_direct_block_filter_mask: 0,
+            offset_size: 8,
+            length_size: 8,
         };
         let (off, len) = fh.decode_managed_id(&id).unwrap();
         assert_eq!(off, 100);

crates/clawhdf5-format/src/filter_pipeline.rs

@@ -19,6 +19,18 @@ pub const FILTER_SCALEOFFSET: u16 = 6;
 pub const FILTER_LZ4: u16 = 32004;
 /// Zstandard compression.
 pub const FILTER_ZSTD: u16 = 32015;
+/// bzip2 (registered by PyTables; hdf5plugin's `BZip2`).
+pub const FILTER_BZIP2: u16 = 307;
+/// LZF — h5py's built-in `compression="lzf"`.
+pub const FILTER_LZF: u16 = 32000;
+/// Blosc 1 (hdf5-blosc; hdf5plugin's `Blosc`).
+pub const FILTER_BLOSC: u16 = 32001;
+/// Bitshuffle, optionally with LZ4 or Zstandard (hdf5plugin's `Bitshuffle`).
+pub const FILTER_BITSHUFFLE: u16 = 32008;
+/// ZFP lossy floating-point compression (hdf5plugin's `Zfp`). Not supported.
+pub const FILTER_ZFP: u16 = 32013;
+/// Blosc 2 (hdf5plugin's `Blosc2`).
+pub const FILTER_BLOSC2: u16 = 32026;
 /// Pcodec lossless numerical codec — a **private, unregistered** clawhdf5
 /// filter. Pcodec has no ID in the HDF Group's filter registry (checked
 /// 2026-09-25, `hdf5_plugins/docs/RegisteredFilterPlugins.md`), so it uses an

crates/clawhdf5-format/src/filter_registry.rs

@@ -0,0 +1,477 @@
+//! Filter registry: every filter is looked up here by its HDF5 filter ID.
+//!
+//! Two tiers:
+//!
+//! * **Built-in filters** — a static table of the filters compiled into this
+//!   build: the HDF5 standard filters (deflate, shuffle, Fletcher32, szip,
+//!   N-Bit, scale-offset) and the plugin filters whose cargo features are
+//!   enabled (LZ4, Zstandard, pcodec, LZF, bitshuffle, bzip2, blosc).
+//!   [`builtin_filters`] lists them.
+//! * **Registered filters** (`std` only) — codecs the application supplies
+//!   for any other ID with [`register_filter`] (a [`FilterCodec`], or just a
+//!   decoding closure). A registered codec cannot shadow a built-in one,
+//!   except under 32023: that ID belongs to Granular BitRound, and the
+//!   built-in entry there only reads the pcodec chunks clawhdf5 <= 2.7.0
+//!   wrote (filter name `"pcodec"`), so a codec registered for 32023 handles
+//!   every other chunk with that ID, and writes.
+//!
+//! An ID in neither tier fails with [`FormatError::UnsupportedFilter`], as it
+//! always has.
+//!
+//! ```
+//! # #[cfg(feature = "std")] {
+//! use clawhdf5_format::filter_registry::{self, FilterContext};
+//! use clawhdf5_format::error::FormatError;
+//!
+//! // A toy filter in the private-use range: every byte XORed with 0x5A.
+//! filter_registry::register_filter(300, |input: &[u8], _ctx: &FilterContext<'_>| {
+//!     Ok::<_, FormatError>(input.iter().map(|b| b ^ 0x5A).collect())
+//! })
+//! .unwrap();
+//! assert!(filter_registry::is_filter_available(300));
+//! filter_registry::unregister_filter(300);
+//! # }
+//! ```
+
+#[cfg(not(feature = "std"))]
+extern crate alloc;
+
+#[cfg(not(feature = "std"))]
+use alloc::vec::Vec;
+
+use crate::error::FormatError;
+use crate::filter_pipeline::FilterDescription;
+
+/// What a codec is told about the filter it is applying.
+#[derive(Debug, Clone, Copy)]
+pub struct FilterContext<'a> {
+    /// The filter as recorded in the dataset's filter pipeline: its ID, name,
+    /// flags and client data (`cd_values`).
+    pub filter: &'a FilterDescription,
+    /// Size in bytes of one dataset element (the datatype's size).
+    pub element_size: usize,
+    /// Decoding only: the most bytes this stage may produce — what entered
+    /// the filter when the chunk was written. 0 means unknown; a decoder then
+    /// falls back to a fixed ceiling. Always 0 when encoding.
+    pub max_output: usize,
+}
+
+impl FilterContext<'_> {
+    /// The filter's client data (`cd_values`).
+    pub fn client_data(&self) -> &[u32] {
+        &self.filter.client_data
+    }
+
+    /// The largest output a decoder should allow: [`Self::max_output`], or
+    /// 256 MiB when that is unknown.
+    pub fn output_limit(&self) -> usize {
+        if self.max_output != 0 {
+            self.max_output
+        } else {
+            crate::filters::MAX_DECOMPRESS_SIZE
+        }
+    }
+}
+
+/// A filter implementation.
+///
+/// `decode` undoes the filter (the read direction). `encode` applies it (the
+/// write direction); the default refuses with
+/// [`FormatError::UnsupportedFilter`], which is right for a read-only codec.
+pub trait FilterCodec: Send + Sync {
+    /// Undo the filter on one chunk. The output must not exceed
+    /// [`FilterContext::output_limit`]; the pipeline rejects a larger one.
+    fn decode(&self, input: &[u8], ctx: &FilterContext<'_>) -> Result<Vec<u8>, FormatError>;
+
+    /// Apply the filter to one chunk.
+    fn encode(&self, input: &[u8], ctx: &FilterContext<'_>) -> Result<Vec<u8>, FormatError> {
+        let _ = input;
+        Err(FormatError::UnsupportedFilter(ctx.filter.filter_id))
+    }
+}
+
+/// Any `Fn(&[u8], &FilterContext) -> Result<Vec<u8>, FormatError>` is a
+/// decode-only codec.
+impl<F> FilterCodec for F
+where
+    F: Fn(&[u8], &FilterContext<'_>) -> Result<Vec<u8>, FormatError> + Send + Sync,
+{
+    fn decode(&self, input: &[u8], ctx: &FilterContext<'_>) -> Result<Vec<u8>, FormatError> {
+        self(input, ctx)
+    }
+}
+
+/// Signature of a built-in filter's decoder or encoder.
+pub type BuiltinFn = fn(&[u8], &FilterContext<'_>) -> Result<Vec<u8>, FormatError>;
+
+/// A filter compiled into this build.
+#[derive(Debug, Clone, Copy)]
+pub struct BuiltinFilter {
+    /// HDF5 filter ID.
+    pub id: u16,
+    /// Human-readable name.
+    pub name: &'static str,
+    /// Decoder.
+    pub(crate) decode: BuiltinFn,
+    /// Encoder, if this build can write the filter.
+    pub(crate) encode: Option<BuiltinFn>,
+}
+
+impl BuiltinFilter {
+    /// Whether this build can write the filter as well as read it.
+    pub fn can_encode(&self) -> bool {
+        self.encode.is_some()
+    }
+
+    /// Whether the built-in entry only borrows its ID for some chunks, so a
+    /// registered codec may take the rest: the legacy pcodec entry under
+    /// Granular BitRound's 32023, which claims only chunks named `"pcodec"`.
+    fn is_shared(&self) -> bool {
+        self.id == crate::filter_pipeline::FILTER_PCODEC_LEGACY
+    }
+
+    /// Whether this entry decodes chunks written with `filter`.
+    fn claims(&self, filter: &crate::filter_pipeline::FilterDescription) -> bool {
+        !self.is_shared()
+            || filter.name.as_deref() == Some(crate::filter_pipeline::FILTER_PCODEC_LEGACY_NAME)
+    }
+}
+
+/// The filters compiled into this build, in ID order.
+pub fn builtin_filters() -> &'static [BuiltinFilter] {
+    crate::filters::BUILTIN_FILTERS
+}
+
+/// The built-in filter with this ID, if it is compiled in.
+pub fn builtin_filter(id: u16) -> Option<&'static BuiltinFilter> {
+    builtin_filters().iter().find(|f| f.id == id)
+}
+
+/// Why a filter ID may be missing from this build: the filter's name, and
+/// the cargo feature that provides it (`None`: clawhdf5 does not implement
+/// it — register a codec for it with [`register_filter`]). `None` for an ID
+/// clawhdf5 knows nothing about.
+pub fn known_filter(id: u16) -> Option<(&'static str, Option<&'static str>)> {
+    Some(match id {
+        1 => ("deflate", Some("deflate")),
+        4 => ("SZIP", Some("szip")),
+        307 => ("bzip2", Some("bzip2")),
+        480 => ("pcodec", Some("pcodec")),
+        32000 => ("LZF", Some("lzf")),
+        32001 => ("Blosc", Some("blosc")),
+        32004 => ("LZ4", Some("lz4")),
+        32008 => ("bitshuffle", Some("bitshuffle")),
+        32013 => ("ZFP", None),
+        32015 => ("Zstandard", Some("zstd")),
+        32019 => ("JPEG", None),
+        32022 => ("BitGroom", None),
+        32023 => ("Granular BitRound", None),
+        32026 => ("Blosc2", None),
+        _ => return None,
+    })
+}
+
+/// Whether a chunk filtered with `id` can be decoded: a built-in filter or a
+/// registered one.
+pub fn is_filter_available(id: u16) -> bool {
+    if builtin_filter(id).is_some() {
+        return true;
+    }
+    #[cfg(feature = "std")]
+    {
+        registered(id).is_some()
+    }
+    #[cfg(not(feature = "std"))]
+    {
+        false
+    }
+}
+
+#[cfg(feature = "std")]
+mod custom {
+    use super::FilterCodec;
+    use std::collections::BTreeMap;
+    use std::sync::{Arc, PoisonError, RwLock};
+
+    pub(super) type Registry = BTreeMap<u16, Arc<dyn FilterCodec>>;
+
+    static REGISTRY: RwLock<Registry> = RwLock::new(BTreeMap::new());
+
+    pub(super) fn with_read<R>(f: impl FnOnce(&Registry) -> R) -> R {
+        // A panic while holding the lock cannot leave the map half-updated
+        // (every update is a single insert/remove), so poisoning is ignored.
+        f(&REGISTRY.read().unwrap_or_else(PoisonError::into_inner))
+    }
+
+    pub(super) fn with_write<R>(f: impl FnOnce(&mut Registry) -> R) -> R {
+        f(&mut REGISTRY.write().unwrap_or_else(PoisonError::into_inner))
+    }
+}
+
+/// Register a codec for filter `id`, process-wide. It is used for every
+/// chunk read (and, if it implements [`FilterCodec::encode`], written) with
+/// that filter ID, by every file.
+///
+/// A plain closure `Fn(&[u8], &FilterContext) -> Result<Vec<u8>, FormatError>`
+/// registers a decoder. Replaces (and returns) an earlier registration for
+/// the same ID. Fails with [`FormatError::FilterError`] if `id` is a built-in
+/// filter of this build: those cannot be overridden. The exception is 32023
+/// (Granular BitRound): with the `pcodec` feature the built-in entry there
+/// reads only chunks whose filter is named `"pcodec"` (clawhdf5 <= 2.7.0's
+/// files); a codec registered for 32023 decodes every other chunk with that
+/// ID and does all the writing.
+#[cfg(feature = "std")]
+pub fn register_filter<C>(
+    id: u16,
+    codec: C,
+) -> Result<Option<std::sync::Arc<dyn FilterCodec>>, FormatError>
+where
+    C: FilterCodec + 'static,
+{
+    if let Some(builtin) = builtin_filter(id).filter(|b| !b.is_shared()) {
+        return Err(FormatError::FilterError(format!(
+            "filter {id} ({}) is built in and cannot be re-registered",
+            builtin.name
+        )));
+    }
+    let codec: std::sync::Arc<dyn FilterCodec> = std::sync::Arc::new(codec);
+    Ok(custom::with_write(|r| r.insert(id, codec)))
+}
+
+/// Remove the codec registered for `id`. Returns whether one was registered.
+#[cfg(feature = "std")]
+pub fn unregister_filter(id: u16) -> bool {
+    custom::with_write(|r| r.remove(&id).is_some())
+}
+
+/// The codec registered for `id`, if any.
+#[cfg(feature = "std")]
+pub fn registered(id: u16) -> Option<std::sync::Arc<dyn FilterCodec>> {
+    custom::with_read(|r| r.get(&id).cloned())
+}
+
+/// Undo filter `ctx.filter` on `input`: the built-in decoder if there is one
+/// that claims the chunk, else a registered one, else the built-in decoder's
+/// own refusal or [`FormatError::UnsupportedFilter`].
+pub(crate) fn decode(input: &[u8], ctx: &FilterContext<'_>) -> Result<Vec<u8>, FormatError> {
+    let id = ctx.filter.filter_id;
+    let builtin = builtin_filter(id);
+    if let Some(builtin) = builtin.filter(|b| b.claims(ctx.filter)) {
+        return (builtin.decode)(input, ctx);
+    }
+    #[cfg(feature = "std")]
+    if let Some(codec) = registered(id) {
+        let out = codec.decode(input, ctx)?;
+        // A registered codec is outside our control: hold it to the same
+        // bound the built-in decoders enforce.
+        if out.len() > ctx.output_limit() {
+            return Err(FormatError::DecompressionError(format!(
+                "filter {id}: decoded {} bytes, more than the {} the chunk can hold",
+                out.len(),
+                ctx.output_limit()
+            )));
+        }
+        return Ok(out);
+    }
+    match builtin {
+        Some(builtin) => (builtin.decode)(input, ctx),
+        None => Err(FormatError::UnsupportedFilter(id)),
+    }
+}
+
+/// Apply filter `ctx.filter` to `input`.
+pub(crate) fn encode(input: &[u8], ctx: &FilterContext<'_>) -> Result<Vec<u8>, FormatError> {
+    let id = ctx.filter.filter_id;
+    #[cfg(feature = "std")]
+    if builtin_filter(id).is_some_and(|b| b.is_shared())
+        && let Some(codec) = registered(id)
+    {
+        return codec.encode(input, ctx);
+    }
+    if let Some(builtin) = builtin_filter(id) {
+        return match builtin.encode {
+            Some(encode) => encode(input, ctx),
+            None => Err(FormatError::UnsupportedFilter(id)),
+        };
+    }
+    #[cfg(feature = "std")]
+    if let Some(codec) = registered(id) {
+        return codec.encode(input, ctx);
+    }
+    Err(FormatError::UnsupportedFilter(id))
+}
+
+#[cfg(all(test, feature = "std"))]
+pub(crate) mod tests {
+    use super::*;
+    use crate::filter_pipeline::{FILTER_FLETCHER32, FILTER_SHUFFLE, FilterPipeline};
+    use crate::filters::{compress_chunk, decompress_chunk};
+
+    fn pipeline(id: u16) -> FilterPipeline {
+        FilterPipeline {
+            version: 2,
+            filters: vec![FilterDescription {
+                filter_id: id,
+                name: Some("test".into()),
+                flags: 0,
+                client_data: vec![7],
+            }],
+        }
+    }
+
+    struct Xor;
+    impl FilterCodec for Xor {
+        fn decode(&self, input: &[u8], ctx: &FilterContext<'_>) -> Result<Vec<u8>, FormatError> {
+            let k = ctx.client_data()[0] as u8;
+            Ok(input.iter().map(|b| b ^ k).collect())
+        }
+        fn encode(&self, input: &[u8], ctx: &FilterContext<'_>) -> Result<Vec<u8>, FormatError> {
+            self.decode(input, ctx)
+        }
+    }
+
+    // Each test uses its own ID: the registry is process-wide and tests run
+    // in parallel.
+
+    #[test]
+    fn unknown_filter_keeps_its_error() {
+        let err = decompress_chunk(b"abc", &pipeline(311), 3, 1).unwrap_err();
+        assert_eq!(err, FormatError::UnsupportedFilter(311));
+        let err = compress_chunk(b"abc", &pipeline(311), 1).unwrap_err();
+        assert_eq!(err, FormatError::UnsupportedFilter(311));
+    }
+
+    #[test]
+    fn registered_codec_round_trips_through_the_pipeline() {
+        assert!(!is_filter_available(312));
+        assert!(register_filter(312, Xor).unwrap().is_none());
+        assert!(is_filter_available(312));
+        let data = b"hello, registry".to_vec();
+        let enc = compress_chunk(&data, &pipeline(312), 1).unwrap();
+        assert_ne!(enc, data);
+        assert_eq!(
+            decompress_chunk(&enc, &pipeline(312), data.len(), 1).unwrap(),
+            data
+        );
+        assert!(unregister_filter(312));
+        assert!(!unregister_filter(312));
+        assert_eq!(
+            decompress_chunk(&enc, &pipeline(312), data.len(), 1).unwrap_err(),
+            FormatError::UnsupportedFilter(312)
+        );
+    }
+
+    #[test]
+    fn closure_registers_a_decoder_only() {
+        register_filter(313, |input: &[u8], _ctx: &FilterContext<'_>| {
+            Ok(input.iter().rev().copied().collect())
+        })
+        .unwrap();
+        assert_eq!(
+            decompress_chunk(b"abc", &pipeline(313), 3, 1).unwrap(),
+            b"cba"
+        );
+        assert_eq!(
+            compress_chunk(b"abc", &pipeline(313), 1).unwrap_err(),
+            FormatError::UnsupportedFilter(313)
+        );
+        unregister_filter(313);
+    }
+
+    #[test]
+    fn registered_decoder_output_is_bounded() {
+        register_filter(314, |_input: &[u8], _ctx: &FilterContext<'_>| {
+            Ok(vec![0u8; 1000])
+        })
+        .unwrap();
+        let err = decompress_chunk(b"abc", &pipeline(314), 10, 1).unwrap_err();
+        assert!(matches!(err, FormatError::DecompressionError(_)), "{err:?}");
+        unregister_filter(314);
+    }
+
+    #[test]
+    fn builtins_cannot_be_overridden() {
+        for id in [FILTER_SHUFFLE, FILTER_FLETCHER32] {
+            let Err(err) = register_filter(id, Xor) else {
+                panic!("built-in filter {id} was re-registered");
+            };
+            assert!(matches!(err, FormatError::FilterError(_)), "{err:?}");
+        }
+        assert!(builtin_filter(FILTER_SHUFFLE).is_some());
+    }
+
+    /// Serialises the tests that register or read filter 32023 (the
+    /// registry is process-wide).
+    pub(crate) static ID_32023: std::sync::Mutex<()> = std::sync::Mutex::new(());
+
+    /// 32023 is Granular BitRound's ID; the `pcodec` build's built-in entry
+    /// there reads only clawhdf5 <= 2.7.0's pcodec chunks (named "pcodec"),
+    /// so a codec can be registered for the rest, and writes with it.
+    #[test]
+    fn a_codec_can_be_registered_for_granular_bitround() {
+        let _guard = ID_32023
+            .lock()
+            .unwrap_or_else(std::sync::PoisonError::into_inner);
+        let named = |name: Option<&str>| FilterPipeline {
+            version: 2,
+            filters: vec![FilterDescription {
+                filter_id: 32023,
+                name: name.map(Into::into),
+                flags: 0,
+                client_data: vec![7],
+            }],
+        };
+        let prev = register_filter(32023, Xor).expect("32023 must be registrable");
+        assert!(prev.is_none());
+        let data = b"granular bitround".to_vec();
+        for name in [None, Some("granular_bitround"), Some("test")] {
+            let pl = named(name);
+            let enc = compress_chunk(&data, &pl, 1).unwrap();
+            assert_ne!(enc, data);
+            assert_eq!(decompress_chunk(&enc, &pl, data.len(), 1).unwrap(), data);
+        }
+        // clawhdf5 <= 2.7.0's pcodec chunks still go to the built-in reader.
+        #[cfg(feature = "pcodec")]
+        {
+            let raw: Vec<u8> = (0..64)
+                .flat_map(|i| (f64::from(i) * 0.5).to_le_bytes())
+                .collect();
+            let comp = crate::filters::pcodec_compress(&raw, 8).unwrap();
+            let mut pl = named(Some("pcodec"));
+            pl.filters[0].client_data = vec![8];
+            assert_eq!(decompress_chunk(&comp, &pl, raw.len(), 8).unwrap(), raw);
+        }
+        assert!(unregister_filter(32023));
+        let pl = named(None);
+        assert!(matches!(
+            decompress_chunk(&data, &pl, data.len(), 1),
+            Err(FormatError::UnsupportedFilter(32023))
+        ));
+    }
+
+    #[test]
+    fn unsupported_filter_error_names_the_filter() {
+        let msg = FormatError::UnsupportedFilter(32026).to_string();
+        assert!(
+            msg.contains("Blosc2") && msg.contains("not implemented"),
+            "{msg}"
+        );
+        let msg = FormatError::UnsupportedFilter(32013).to_string();
+        assert!(msg.contains("ZFP"), "{msg}");
+        let msg = FormatError::UnsupportedFilter(32000).to_string();
+        assert!(msg.contains("LZF") && msg.contains("`lzf`"), "{msg}");
+        assert_eq!(
+            FormatError::UnsupportedFilter(399).to_string(),
+            "unsupported filter: 399"
+        );
+    }
+
+    #[test]
+    fn builtin_table_is_sorted_and_unique() {
+        let ids: Vec<u16> = builtin_filters().iter().map(|f| f.id).collect();
+        let mut sorted = ids.clone();
+        sorted.sort_unstable();
+        sorted.dedup();
+        assert_eq!(ids, sorted);
+    }
+}

crates/clawhdf5-format/src/filters.rs

@@ -4,14 +4,23 @@
 extern crate alloc;
 
 #[cfg(not(feature = "std"))]
-use alloc::{boxed::Box, vec, vec::Vec};
+use alloc::{boxed::Box, format, vec, vec::Vec};
 
 use crate::error::FormatError;
+#[cfg(feature = "deflate")]
+use crate::filter_pipeline::FILTER_DEFLATE;
+#[cfg(feature = "lz4")]
+use crate::filter_pipeline::FILTER_LZ4;
+#[cfg(feature = "szip")]
+use crate::filter_pipeline::FILTER_SZIP;
+#[cfg(feature = "zstd")]
+use crate::filter_pipeline::FILTER_ZSTD;
 use crate::filter_pipeline::{
-    FILTER_DEFLATE, FILTER_FLETCHER32, FILTER_LZ4, FILTER_NBIT, FILTER_PCODEC,
+    FILTER_FLETCHER32, FILTER_NBIT, FILTER_SCALEOFFSET, FILTER_SHUFFLE, FilterPipeline,
-    FILTER_PCODEC_LEGACY, FILTER_PCODEC_LEGACY_NAME, FILTER_SCALEOFFSET, FILTER_SHUFFLE,
-    FILTER_SZIP, FILTER_ZSTD, FilterPipeline,
 };
+#[cfg(feature = "pcodec")]
+use crate::filter_pipeline::{FILTER_PCODEC, FILTER_PCODEC_LEGACY, FILTER_PCODEC_LEGACY_NAME};
+use crate::filter_registry::{self, BuiltinFilter, FilterContext};
 
 /// Absolute ceiling on a single decompressed chunk's output size, used only
 /// when the pipeline's declared `chunk_size` is unavailable (0). Prevents
@@ -98,33 +107,47 @@ pub fn decompress_chunk_masked(
         if filter_skipped(filter_mask, i) {
             continue;
         }
-        let bound = bounds[i];
+        // `max_output` caps the decoded size so a decoder can't be forced
-        data = match filter.filter_id {
-            FILTER_SHUFFLE => shuffle_decompress(&data, element_size as usize)?,
-            // `bound` caps the decoded size so these decoders can't be forced
         // into unbounded allocation by a hostile or corrupted payload.
-            FILTER_DEFLATE => deflate_decompress(&data, bound)?,
+        let ctx = FilterContext {
-            FILTER_LZ4 => lz4_decompress(&data, bound)?,
+            filter,
-            FILTER_ZSTD => zstd_decompress(&data, bound)?,
+            element_size: element_size as usize,
-            FILTER_FLETCHER32 => fletcher32_verify(&data)?,
+            max_output: bounds[i],
-            FILTER_PCODEC => pcodec_decompress(&data, element_size as usize, bound)?,
-            // Pcodec chunks written by clawhdf5 <= 2.7.0 under the ID registered
-            // to Granular BitRound; recognised by the name those versions wrote.
-            FILTER_PCODEC_LEGACY if filter.name.as_deref() == Some(FILTER_PCODEC_LEGACY_NAME) => {
-                pcodec_decompress(&data, element_size as usize, bound)?
-            }
-            // These decoders also reject an element count that would
-            // over-allocate past `bound`.
-            FILTER_SCALEOFFSET => scaleoffset_decompress(&data, &filter.client_data, bound)?,
-            FILTER_NBIT => nbit_decompress(&data, &filter.client_data, bound)?,
-            FILTER_SZIP => crate::filters_szip::szip_decompress(&data, &filter.client_data, bound)?,
-            other => return Err(FormatError::UnsupportedFilter(other)),
         };
+        data = filter_registry::decode(&data, &ctx)?;
     }
 
     Ok(data)
 }
 
+/// Decode one stored chunk of a chunked dataset: [`decompress_chunk_masked`],
+/// then require exactly `chunk_size` bytes (when `chunk_size` is known).
+///
+/// HDF5 stores every chunk at the full chunk size — edge chunks are padded
+/// before they are filtered, and an edge chunk left unfiltered is written
+/// full-size too — so a pipeline that decodes to fewer bytes means a
+/// corrupt chunk. libhdf5 fails such a read (or returns uninitialised
+/// memory); it must never read back as zeros. `coords` (the chunk's offset
+/// in the dataset) is named in the error.
+pub fn decompress_chunk_exact(
+    compressed: &[u8],
+    pipeline: &FilterPipeline,
+    chunk_size: usize,
+    element_size: u32,
+    filter_mask: u32,
+    coords: &[u64],
+) -> Result<Vec<u8>, FormatError> {
+    let data =
+        decompress_chunk_masked(compressed, pipeline, chunk_size, element_size, filter_mask)?;
+    if chunk_size != 0 && data.len() != chunk_size {
+        return Err(FormatError::ChunkedReadError(format!(
+            "chunk at {coords:?} decoded to {} bytes, expected {chunk_size}",
+            data.len()
+        )));
+    }
+    Ok(data)
+}
+
 /// Apply a filter pipeline to compress a chunk.
 /// Filters are applied in FORWARD order for compression.
 pub fn compress_chunk(
@@ -135,26 +158,128 @@ pub fn compress_chunk(
     let mut result = data.to_vec();
 
     for filter in &pipeline.filters {
-        result = match filter.filter_id {
+        let ctx = FilterContext {
-            FILTER_SHUFFLE => shuffle_compress(&result, element_size as usize)?,
+            filter,
-            FILTER_DEFLATE => {
+            element_size: element_size as usize,
-                let level = filter.client_data.first().copied().unwrap_or(6);
+            max_output: 0,
-                deflate_compress(&result, level)?
-            }
-            FILTER_LZ4 => lz4_compress(&result, &filter.client_data)?,
-            FILTER_ZSTD => {
-                let level = filter.client_data.first().copied().unwrap_or(3);
-                zstd_compress(&result, level)?
-            }
-            FILTER_FLETCHER32 => fletcher32_append(&result)?,
-            FILTER_PCODEC => pcodec_compress(&result, element_size as usize)?,
-            other => return Err(FormatError::UnsupportedFilter(other)),
         };
+        result = filter_registry::encode(&result, &ctx)?;
     }
 
     Ok(result)
 }
 
+/// The filters compiled into this build, sorted by ID (see
+/// [`crate::filter_registry`]). A filter whose cargo feature is off is left
+/// out, so it fails as [`FormatError::UnsupportedFilter`] like any unknown ID.
+pub(crate) static BUILTIN_FILTERS: &[BuiltinFilter] = &[
+    #[cfg(feature = "deflate")]
+    BuiltinFilter {
+        id: FILTER_DEFLATE,
+        name: "deflate",
+        decode: |d, c| deflate_decompress(d, c.max_output),
+        encode: Some(|d, c| deflate_compress(d, c.client_data().first().copied().unwrap_or(6))),
+    },
+    BuiltinFilter {
+        id: FILTER_SHUFFLE,
+        name: "shuffle",
+        decode: |d, c| shuffle_decompress(d, c.element_size),
+        encode: Some(|d, c| shuffle_compress(d, c.element_size)),
+    },
+    BuiltinFilter {
+        id: FILTER_FLETCHER32,
+        name: "fletcher32",
+        decode: |d, _| fletcher32_verify(d),
+        encode: Some(|d, _| fletcher32_append(d)),
+    },
+    #[cfg(feature = "szip")]
+    BuiltinFilter {
+        id: FILTER_SZIP,
+        name: "szip",
+        decode: |d, c| crate::filters_szip::szip_decompress(d, c.client_data(), c.max_output),
+        encode: None,
+    },
+    // These decoders also reject an element count that would over-allocate
+    // past `max_output`.
+    BuiltinFilter {
+        id: FILTER_NBIT,
+        name: "nbit",
+        decode: |d, c| nbit_decompress(d, c.client_data(), c.max_output),
+        encode: None,
+    },
+    BuiltinFilter {
+        id: FILTER_SCALEOFFSET,
+        name: "scaleoffset",
+        decode: |d, c| scaleoffset_decompress(d, c.client_data(), c.max_output),
+        encode: None,
+    },
+    #[cfg(feature = "bzip2")]
+    BuiltinFilter {
+        id: crate::filter_pipeline::FILTER_BZIP2,
+        name: "bzip2",
+        decode: crate::filters_bzip2::bzip2_decode,
+        encode: Some(crate::filters_bzip2::bzip2_encode),
+    },
+    #[cfg(feature = "pcodec")]
+    BuiltinFilter {
+        id: FILTER_PCODEC,
+        name: "pcodec (clawhdf5 private)",
+        decode: |d, c| pcodec_decompress(d, c.element_size, c.max_output),
+        encode: Some(|d, c| pcodec_compress(d, c.element_size)),
+    },
+    #[cfg(feature = "lzf")]
+    BuiltinFilter {
+        id: crate::filter_pipeline::FILTER_LZF,
+        name: "lzf",
+        decode: crate::filters_lzf::lzf_decode,
+        encode: Some(crate::filters_lzf::lzf_encode),
+    },
+    #[cfg(feature = "blosc")]
+    BuiltinFilter {
+        id: crate::filter_pipeline::FILTER_BLOSC,
+        name: "blosc",
+        decode: crate::filters_blosc::blosc_decode,
+        encode: Some(crate::filters_blosc::blosc_encode),
+    },
+    #[cfg(feature = "lz4")]
+    BuiltinFilter {
+        id: FILTER_LZ4,
+        name: "lz4",
+        decode: |d, c| lz4_decompress(d, c.max_output),
+        encode: Some(|d, c| lz4_compress(d, c.client_data())),
+    },
+    #[cfg(feature = "bitshuffle")]
+    BuiltinFilter {
+        id: crate::filter_pipeline::FILTER_BITSHUFFLE,
+        name: "bitshuffle",
+        decode: crate::filters_bitshuffle::bitshuffle_decode,
+        encode: Some(crate::filters_bitshuffle::bitshuffle_encode),
+    },
+    #[cfg(feature = "zstd")]
+    BuiltinFilter {
+        id: FILTER_ZSTD,
+        name: "zstd",
+        decode: |d, c| zstd_decompress(d, c.max_output),
+        encode: Some(|d, c| zstd_compress(d, c.client_data().first().copied().unwrap_or(3))),
+    },
+    // Pcodec chunks written by clawhdf5 <= 2.7.0 under the ID registered to
+    // Granular BitRound; recognised by the name those versions wrote, and
+    // never written.
+    #[cfg(feature = "pcodec")]
+    BuiltinFilter {
+        id: FILTER_PCODEC_LEGACY,
+        name: "pcodec (clawhdf5 <= 2.7.0)",
+        decode: |d, c| {
+            if c.filter.name.as_deref() == Some(FILTER_PCODEC_LEGACY_NAME) {
+                pcodec_decompress(d, c.element_size, c.max_output)
+            } else {
+                Err(FormatError::UnsupportedFilter(FILTER_PCODEC_LEGACY))
+            }
+        },
+        encode: None,
+    },
+];
+
 /// Decode the HDF5 scale-offset filter (id 6).
 ///
 /// Supports all three scale-offset variants:
@@ -877,11 +1002,6 @@ mod sysz {
     }
 }
 
-#[cfg(not(feature = "deflate"))]
-fn deflate_decompress(_data: &[u8], _expected_bytes: usize) -> Result<Vec<u8>, FormatError> {
-    Err(FormatError::UnsupportedFilter(FILTER_DEFLATE))
-}
-
 /// Compress data with zlib.
 #[cfg(feature = "deflate")]
 fn deflate_compress(data: &[u8], level: u32) -> Result<Vec<u8>, FormatError> {
@@ -922,11 +1042,6 @@ pub(crate) fn deflate_bounded(data: &[u8], level: u32) -> Result<Vec<u8>, String
     }
 }
 
-#[cfg(not(feature = "deflate"))]
-fn deflate_compress(_data: &[u8], _level: u32) -> Result<Vec<u8>, FormatError> {
-    Err(FormatError::UnsupportedFilter(FILTER_DEFLATE))
-}
-
 /// Default LZ4 block size of the registered HDF5 LZ4 filter (`H5Zlz4.c`,
 /// `DEFAULT_BLOCK_SIZE`): 1 GiB, so an HDF5 chunk is normally one block.
 #[cfg(feature = "lz4")]
@@ -1028,11 +1143,6 @@ fn lz4_decompress_hdf5(
     Ok(out)
 }
 
-#[cfg(not(feature = "lz4"))]
-fn lz4_decompress(_data: &[u8], _expected_bytes: usize) -> Result<Vec<u8>, FormatError> {
-    Err(FormatError::UnsupportedFilter(FILTER_LZ4))
-}
-
 /// Compress data in the registered HDF5 LZ4 filter format (see
 /// [`lz4_decompress`]), so libhdf5 with the LZ4 plugin (e.g. hdf5plugin) can
 /// read it. `cd[0]`, when present and non-zero, is the block size in bytes,
@@ -1064,11 +1174,6 @@ fn lz4_compress(data: &[u8], cd: &[u32]) -> Result<Vec<u8>, FormatError> {
     Ok(result)
 }
 
-#[cfg(not(feature = "lz4"))]
-fn lz4_compress(_data: &[u8], _cd: &[u32]) -> Result<Vec<u8>, FormatError> {
-    Err(FormatError::UnsupportedFilter(FILTER_LZ4))
-}
-
 /// Decompress zstd data.
 ///
 /// `expected_bytes` bounds the output (or [`MAX_DECOMPRESS_SIZE`] when
@@ -1097,11 +1202,6 @@ fn zstd_decompress(data: &[u8], expected_bytes: usize) -> Result<Vec<u8>, Format
     Ok(out)
 }
 
-#[cfg(not(feature = "zstd"))]
-fn zstd_decompress(_data: &[u8], _expected_bytes: usize) -> Result<Vec<u8>, FormatError> {
-    Err(FormatError::UnsupportedFilter(FILTER_ZSTD))
-}
-
 /// Compress data with zstd as one frame whose header records the content
 /// size. The registered HDF5 Zstandard filter (`H5Zzstd.c`, used by
 /// libhdf5 + hdf5plugin) sizes its output buffer from
@@ -1113,11 +1213,6 @@ fn zstd_compress(data: &[u8], level: u32) -> Result<Vec<u8>, FormatError> {
         .map_err(|e| FormatError::CompressionError(format!("zstd: {e}")))
 }
 
-#[cfg(not(feature = "zstd"))]
-fn zstd_compress(_data: &[u8], _level: u32) -> Result<Vec<u8>, FormatError> {
-    Err(FormatError::UnsupportedFilter(FILTER_ZSTD))
-}
-
 /// Unshuffle (decompress direction): reconstruct interleaved element bytes.
 /// On disk: all byte-0s of each element together, then all byte-1s, etc.
 /// Output: elements in natural order.
@@ -1351,7 +1446,7 @@ fn fletcher32_append(data: &[u8]) -> Result<Vec<u8>, FormatError> {
 // ---------------------------------------------------------------------------
 
 #[cfg(feature = "pcodec")]
-fn pcodec_compress(data: &[u8], element_size: usize) -> Result<Vec<u8>, FormatError> {
+pub(crate) fn pcodec_compress(data: &[u8], element_size: usize) -> Result<Vec<u8>, FormatError> {
     use pco::ChunkConfig;
     use pco::standalone::simple_compress;
     let config = ChunkConfig::default();
@@ -1389,11 +1484,6 @@ fn pcodec_compress(data: &[u8], element_size: usize) -> Result<Vec<u8>, FormatEr
     }
 }
 
-#[cfg(not(feature = "pcodec"))]
-fn pcodec_compress(_data: &[u8], _element_size: usize) -> Result<Vec<u8>, FormatError> {
-    Err(FormatError::UnsupportedFilter(FILTER_PCODEC))
-}
-
 /// `expected_bytes` bounds the number of elements decoded: the output buffer
 /// is pre-sized to exactly `expected_bytes / element_size` elements and
 /// `simple_decompress_into` never writes past it, so a corrupted/hostile pco
@@ -1452,17 +1542,41 @@ fn pcodec_decompress(
     }
 }
 
-#[cfg(not(feature = "pcodec"))]
-fn pcodec_decompress(
-    _data: &[u8],
-    _element_size: usize,
-    _expected_bytes: usize,
-) -> Result<Vec<u8>, FormatError> {
-    Err(FormatError::UnsupportedFilter(FILTER_PCODEC))
-}
-
 #[cfg(test)]
 mod tests {
+
+    /// A chunk whose pipeline decodes to fewer bytes than the chunk holds is
+    /// an error naming the chunk, never a short buffer the reader pads.
+    #[test]
+    fn short_decoded_chunk_is_an_error() {
+        let pipeline = FilterPipeline {
+            version: 2,
+            filters: vec![FilterDescription {
+                filter_id: FILTER_SHUFFLE,
+                name: None,
+                flags: 0,
+                client_data: vec![4],
+            }],
+        };
+        let full = [7u8; 32];
+        assert_eq!(
+            decompress_chunk_exact(&full, &pipeline, 32, 4, 0, &[8]).unwrap(),
+            full
+        );
+        let err = decompress_chunk_exact(&full[..16], &pipeline, 32, 4, 0, &[8, 0]).unwrap_err();
+        let msg = err.to_string();
+        assert!(
+            msg.contains("[8, 0]") && msg.contains("16") && msg.contains("32"),
+            "{msg}"
+        );
+        // Every filter skipped: the stored bytes are the chunk, still checked.
+        assert!(decompress_chunk_exact(&full[..16], &pipeline, 32, 4, 1, &[0]).is_err());
+        // Unknown chunk size: not checked.
+        assert_eq!(
+            decompress_chunk_exact(&full[..16], &pipeline, 0, 4, 0, &[0]).unwrap(),
+            &full[..16]
+        );
+    }
     use super::*;
     use crate::filter_pipeline::FilterDescription;
 
@@ -1677,7 +1791,7 @@ mod tests {
         // An unsupported filter is fine when the chunk skipped it.
         let unknown = FilterPipeline {
             version: 2,
-            filters: vec![filter(32000), filter(FILTER_DEFLATE)],
+            filters: vec![filter(32013), filter(FILTER_DEFLATE)],
         };
         assert_eq!(
             decompress_chunk_masked(&deflated, &unknown, n, 8, 0b01).unwrap(),
@@ -2644,6 +2758,10 @@ mod tests {
     #[cfg(feature = "pcodec")]
     fn pcodec_uses_private_id_and_reads_legacy_32023() {
         use crate::chunked_write::ChunkOptions;
+        #[cfg(feature = "std")]
+        let _guard = crate::filter_registry::tests::ID_32023
+            .lock()
+            .unwrap_or_else(std::sync::PoisonError::into_inner);
         let opts = ChunkOptions {
             pcodec: true,
             ..Default::default()

crates/clawhdf5-format/src/filters_bitshuffle.rs

@@ -0,0 +1,438 @@
+//! Bitshuffle (HDF5 filter 32008) and the bit transpose it shares with blosc.
+//!
+//! **The transform.** A block of `n` elements (`n` a multiple of 8) of
+//! `es` bytes each is viewed as an `n × 8·es` bit matrix — row *i* is
+//! element *i*, column `8·j + k` is bit *k* (LSB first) of its byte *j* — and
+//! transposed: the output is `8·es` rows of `n` bits, row `8·j + k` holding
+//! bit *k* of byte *j* of every element in order, packed LSB first. That is
+//! what `bshuf_trans_bit_elem` produces (checked against hdf5plugin's
+//! library bit for bit).
+//!
+//! **The filter** (`bshuf_h5filter.c`). `cd_values`: `[0..2]` bitshuffle
+//! version, `[2]` element size, `[3]` block size in elements (0 = default:
+//! 8192 bytes' worth, rounded down to a multiple of 8, at least 128),
+//! `[4]` compression (0 none, 2 LZ4, 3 Zstandard), `[5]` Zstandard level.
+//! The chunk is cut into blocks of `block size` elements; the tail shorter
+//! than a block is transposed as one block rounded down to a multiple of 8
+//! elements, and the last `n mod 8` elements are stored as they are.
+//! Uncompressed, that is the whole chunk. Compressed, the chunk starts with a
+//! 12-byte header — the decoded size (u64 big-endian) and the block size in
+//! bytes (u32 big-endian) — and each transposed block is stored as a u32
+//! big-endian length and an LZ4 block / Zstandard frame; the untransposed
+//! tail follows the last block.
+
+#[cfg(not(feature = "std"))]
+extern crate alloc;
+#[cfg(not(feature = "std"))]
+use alloc::{format, vec, vec::Vec};
+
+use crate::error::FormatError;
+#[cfg(feature = "bitshuffle")]
+use crate::filter_registry::FilterContext;
+
+/// Transpose an 8×8 bit matrix packed in a u64 (byte *r* = row *r*, bit *c*
+/// of that byte = column *c*). An involution.
+#[inline]
+fn transpose8(mut x: u64) -> u64 {
+    let t = (x ^ (x >> 7)) & 0x00AA_00AA_00AA_00AA;
+    x = x ^ t ^ (t << 7);
+    let t = (x ^ (x >> 14)) & 0x0000_CCCC_0000_CCCC;
+    x = x ^ t ^ (t << 14);
+    let t = (x ^ (x >> 28)) & 0x0000_0000_F0F0_F0F0;
+    x ^ t ^ (t << 28)
+}
+
+/// Bit-transpose one block: `input` and `out` are `n * es` bytes, `n` a
+/// multiple of 8.
+pub(crate) fn bitshuffle_block(input: &[u8], out: &mut [u8], n: usize, es: usize) {
+    debug_assert!(n.is_multiple_of(8) && input.len() == n * es && out.len() == n * es);
+    let row = n / 8;
+    for j in 0..es {
+        for g in 0..row {
+            let mut x = 0u64;
+            for t in 0..8 {
+                x |= u64::from(input[(8 * g + t) * es + j]) << (8 * t);
+            }
+            let y = transpose8(x);
+            for k in 0..8 {
+                out[(8 * j + k) * row + g] = (y >> (8 * k)) as u8;
+            }
+        }
+    }
+}
+
+/// Undo [`bitshuffle_block`].
+pub(crate) fn bitunshuffle_block(input: &[u8], out: &mut [u8], n: usize, es: usize) {
+    debug_assert!(n.is_multiple_of(8) && input.len() == n * es && out.len() == n * es);
+    let row = n / 8;
+    for j in 0..es {
+        for g in 0..row {
+            let mut y = 0u64;
+            for k in 0..8 {
+                y |= u64::from(input[(8 * j + k) * row + g]) << (8 * k);
+            }
+            let x = transpose8(y);
+            for t in 0..8 {
+                out[(8 * g + t) * es + j] = (x >> (8 * t)) as u8;
+            }
+        }
+    }
+}
+
+/// `bshuf_default_block_size`: 8 KiB of elements, a multiple of 8, >= 128.
+#[cfg(feature = "bitshuffle")]
+fn default_block_size(es: usize) -> usize {
+    ((8192 / es) / 8 * 8).max(128)
+}
+
+#[cfg(feature = "bitshuffle")]
+fn err(msg: &str) -> FormatError {
+    FormatError::DecompressionError(format!("bitshuffle: {msg}"))
+}
+
+/// `cd_values[4]`: the compression bitshuffle applies after the transpose.
+#[cfg(feature = "bitshuffle")]
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum Codec {
+    None,
+    Lz4,
+    Zstd,
+}
+
+#[cfg(feature = "bitshuffle")]
+fn codec(cd: &[u32]) -> Result<Codec, FormatError> {
+    match cd.get(4).copied().unwrap_or(0) {
+        0 => Ok(Codec::None),
+        2 => Ok(Codec::Lz4),
+        3 => Ok(Codec::Zstd),
+        other => Err(FormatError::FilterError(format!(
+            "bitshuffle: unknown compression {other}"
+        ))),
+    }
+}
+
+/// The element counts of the transposed blocks for `size` elements.
+#[cfg(feature = "bitshuffle")]
+fn blocks(size: usize, block: usize) -> impl Iterator<Item = usize> {
+    let full = size / block;
+    let last = (size % block) / 8 * 8;
+    core::iter::repeat_n(block, full).chain((last > 0).then_some(last))
+}
+
+/// Decode a bitshuffle-filtered chunk.
+#[cfg(feature = "bitshuffle")]
+pub(crate) fn bitshuffle_decode(
+    input: &[u8],
+    ctx: &FilterContext<'_>,
+) -> Result<Vec<u8>, FormatError> {
+    let cd = ctx.client_data();
+    let es = match cd.get(2) {
+        Some(&e) if e != 0 => e as usize,
+        _ => return Err(err("missing element size")),
+    };
+    let codec = codec(cd)?;
+    let limit = ctx.output_limit();
+    if codec == Codec::None {
+        if input.len() > limit {
+            return Err(err("output exceeds the chunk size"));
+        }
+        let block = match cd.get(3) {
+            Some(&b) if b != 0 => b as usize,
+            _ => default_block_size(es),
+        };
+        if !block.is_multiple_of(8) {
+            return Err(err("block size is not a multiple of 8"));
+        }
+        if !input.len().is_multiple_of(es) {
+            return Err(err("chunk is not a whole number of elements"));
+        }
+        let size = input.len() / es;
+        let mut out = vec![0u8; input.len()];
+        let mut pos = 0;
+        for n in blocks(size, block) {
+            let bytes = n * es;
+            bitunshuffle_block(&input[pos..pos + bytes], &mut out[pos..pos + bytes], n, es);
+            pos += bytes;
+        }
+        out[pos..].copy_from_slice(&input[pos..]);
+        return Ok(out);
+    }
+
+    let header = input.get(..12).ok_or_else(|| err("truncated header"))?;
+    let total = u64::from_be_bytes(header[..8].try_into().unwrap());
+    let block_bytes = u32::from_be_bytes(header[8..12].try_into().unwrap()) as usize;
+    let total = usize::try_from(total)
+        .ok()
+        .filter(|&t| t <= limit)
+        .ok_or_else(|| err("decoded size exceeds the chunk size"))?;
+    if !total.is_multiple_of(es) {
+        return Err(err("chunk is not a whole number of elements"));
+    }
+    if block_bytes == 0 || !block_bytes.is_multiple_of(es) {
+        return Err(err("bad block size"));
+    }
+    let block = block_bytes / es;
+    if !block.is_multiple_of(8) {
+        return Err(err("block size is not a multiple of 8"));
+    }
+    let size = total / es;
+    let mut out = vec![0u8; total];
+    let mut tmp = vec![0u8; block_bytes.min(total)];
+    let mut ip = 12usize;
+    let mut op = 0usize;
+    let mut zstd = None;
+    for n in blocks(size, block) {
+        let bytes = n * es;
+        let len = input
+            .get(ip..ip + 4)
+            .map(|b| u32::from_be_bytes(b.try_into().unwrap()) as usize)
+            .ok_or_else(|| err("truncated block header"))?;
+        ip += 4;
+        let comp = input
+            .get(ip..ip.saturating_add(len))
+            .ok_or_else(|| err("truncated block"))?;
+        ip += len;
+        let dst = &mut tmp[..bytes];
+        let got = match codec {
+            Codec::Lz4 => lz4_flex::block::decompress_into(comp, dst)
+                .map_err(|e| err(&format!("lz4: {e}")))?,
+            Codec::Zstd => zstd_decode_into(
+                zstd.get_or_insert_with(ruzstd::decoding::FrameDecoder::new),
+                comp,
+                dst,
+            )?,
+            Codec::None => unreachable!(),
+        };
+        if got != bytes {
+            return Err(err("block decoded to the wrong size"));
+        }
+        bitunshuffle_block(dst, &mut out[op..op + bytes], n, es);
+        op += bytes;
+    }
+    let tail = total - op;
+    let rest = input
+        .get(ip..ip + tail)
+        .ok_or_else(|| err("truncated trailing elements"))?;
+    out[op..].copy_from_slice(rest);
+    Ok(out)
+}
+
+/// Decode Zstandard frames into exactly `dst`, failing if they hold more.
+#[cfg(any(feature = "bitshuffle", feature = "blosc"))]
+pub(crate) fn zstd_decode_into(
+    decoder: &mut ruzstd::decoding::FrameDecoder,
+    frames: &[u8],
+    dst: &mut [u8],
+) -> Result<usize, FormatError> {
+    decoder
+        .decode_all(frames, dst)
+        .map_err(|e| FormatError::DecompressionError(format!("zstd: {e}")))
+}
+
+/// Compress with ruzstd. It implements one level (roughly zstd's level 1),
+/// so the requested level only matters to other encoders.
+#[cfg(any(feature = "bitshuffle", feature = "blosc"))]
+pub(crate) fn zstd_encode(data: &[u8]) -> Vec<u8> {
+    ruzstd::encoding::compress_to_vec(data, ruzstd::encoding::CompressionLevel::Fastest)
+}
+
+/// Encode a chunk with the bitshuffle filter.
+#[cfg(feature = "bitshuffle")]
+pub(crate) fn bitshuffle_encode(
+    input: &[u8],
+    ctx: &FilterContext<'_>,
+) -> Result<Vec<u8>, FormatError> {
+    let cd = ctx.client_data();
+    let es = match cd.get(2) {
+        Some(&e) if e != 0 => e as usize,
+        _ => ctx.element_size.max(1),
+    };
+    let codec = codec(cd)?;
+    let block = match cd.get(3) {
+        Some(&b) if b != 0 => b as usize,
+        _ => default_block_size(es),
+    };
+    let cerr = |m: &str| FormatError::CompressionError(format!("bitshuffle: {m}"));
+    if !block.is_multiple_of(8) {
+        return Err(cerr("block size is not a multiple of 8"));
+    }
+    if !input.len().is_multiple_of(es) {
+        return Err(cerr("chunk is not a whole number of elements"));
+    }
+    let size = input.len() / es;
+    let mut out = Vec::with_capacity(input.len() + 12 + input.len() / 64);
+    if codec != Codec::None {
+        out.extend_from_slice(&(input.len() as u64).to_be_bytes());
+        let block_bytes =
+            u32::try_from(block * es).map_err(|_| cerr("block size does not fit in 32 bits"))?;
+        out.extend_from_slice(&block_bytes.to_be_bytes());
+    }
+    let mut tmp = vec![0u8; (block * es).min(input.len())];
+    let mut pos = 0;
+    for n in blocks(size, block) {
+        let bytes = n * es;
+        let dst = &mut tmp[..bytes];
+        bitshuffle_block(&input[pos..pos + bytes], dst, n, es);
+        match codec {
+            Codec::None => out.extend_from_slice(dst),
+            Codec::Lz4 | Codec::Zstd => {
+                let comp = if codec == Codec::Lz4 {
+                    lz4_flex::block::compress(dst)
+                } else {
+                    zstd_encode(dst)
+                };
+                out.extend_from_slice(&(comp.len() as u32).to_be_bytes());
+                out.extend_from_slice(&comp);
+            }
+        }
+        pos += bytes;
+    }
+    out.extend_from_slice(&input[pos..]);
+    Ok(out)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    /// The definition, one bit at a time.
+    fn naive(input: &[u8], n: usize, es: usize) -> Vec<u8> {
+        let mut out = vec![0u8; n * es];
+        for i in 0..n {
+            for j in 0..es {
+                for k in 0..8 {
+                    if input[i * es + j] >> k & 1 == 1 {
+                        let p = (8 * j + k) * n + i;
+                        out[p / 8] |= 1 << (p % 8);
+                    }
+                }
+            }
+        }
+        out
+    }
+
+    #[test]
+    fn transpose_matches_the_definition_and_inverts() {
+        for (n, es) in [(8, 1), (16, 2), (24, 4), (128, 8), (64, 3), (8, 16)] {
+            let input: Vec<u8> = (0..n * es)
+                .map(|i| (i as u32).wrapping_mul(2_654_435_761).rotate_left(7) as u8)
+                .collect();
+            let mut out = vec![0u8; n * es];
+            bitshuffle_block(&input, &mut out, n, es);
+            assert_eq!(out, naive(&input, n, es), "n={n} es={es}");
+            let mut back = vec![0u8; n * es];
+            bitunshuffle_block(&out, &mut back, n, es);
+            assert_eq!(back, input);
+        }
+    }
+
+    #[cfg(feature = "bitshuffle")]
+    fn ctx_for(cd: Vec<u32>) -> crate::filter_pipeline::FilterDescription {
+        crate::filter_pipeline::FilterDescription {
+            filter_id: crate::filter_pipeline::FILTER_BITSHUFFLE,
+            name: None,
+            flags: 0,
+            client_data: cd,
+        }
+    }
+
+    #[cfg(feature = "bitshuffle")]
+    #[test]
+    fn filter_round_trips_every_mode() {
+        for es in [1usize, 2, 4, 8] {
+            for n in [0usize, 1, 7, 8, 100, 1000, 5003] {
+                let data: Vec<u8> = (0..n * es)
+                    .map(|i| (i % 97) as u8 ^ (i / 300) as u8)
+                    .collect();
+                for (comp, block) in [(0, 0), (0, 16), (2, 0), (2, 64), (3, 0), (3, 1024)] {
+                    let f = ctx_for(vec![0, 4, es as u32, block, comp]);
+                    let ctx = FilterContext {
+                        filter: &f,
+                        element_size: es,
+                        max_output: data.len(),
+                    };
+                    let enc = bitshuffle_encode(&data, &ctx).unwrap();
+                    let dec = bitshuffle_decode(&enc, &ctx).unwrap();
+                    assert_eq!(dec, data, "es={es} n={n} comp={comp} block={block}");
+                }
+            }
+        }
+    }
+
+    #[cfg(feature = "bitshuffle")]
+    #[test]
+    fn rejects_oversized_and_truncated_chunks() {
+        let data = vec![5u8; 4096];
+        let f = ctx_for(vec![0, 4, 4, 0, 2]);
+        let mut ctx = FilterContext {
+            filter: &f,
+            element_size: 4,
+            max_output: data.len(),
+        };
+        let enc = bitshuffle_encode(&data, &ctx).unwrap();
+        assert!(bitshuffle_decode(&enc[..enc.len() - 1], &ctx).is_err());
+        ctx.max_output = 100;
+        assert!(bitshuffle_decode(&enc, &ctx).is_err());
+    }
+
+    /// Random and mutated chunks, in every mode, and hostile `cd_values`:
+    /// errors are fine, panics are not.
+    #[cfg(feature = "bitshuffle")]
+    #[test]
+    fn fuzzed_chunks_never_panic() {
+        use crate::test_fuzz::{Rng, fuzz_decoder};
+        let data: Vec<u8> = (0..3001u32)
+            .flat_map(|i| ((i / 7) as u16).to_le_bytes())
+            .collect();
+        for (comp, block) in [(0, 0), (0, 16), (2, 0), (2, 64), (3, 0), (3, 1024)] {
+            let f = ctx_for(vec![0, 4, 2, block, comp]);
+            let ctx = FilterContext {
+                filter: &f,
+                element_size: 2,
+                max_output: data.len(),
+            };
+            let seeds = vec![
+                bitshuffle_encode(&data, &ctx).unwrap(),
+                bitshuffle_encode(&data[..34], &ctx).unwrap(),
+                bitshuffle_encode(&data[..512], &ctx).unwrap(),
+            ];
+            fuzz_decoder(
+                0xb5 + comp as u64 * 7 + block as u64,
+                &seeds,
+                4_000,
+                data.len(),
+                |s| bitshuffle_decode(s, &ctx),
+            );
+        }
+        // Hostile filter parameters on a valid chunk.
+        let mut rng = Rng::new(0xcd);
+        let good = ctx_for(vec![0, 4, 2, 0, 2]);
+        let enc = bitshuffle_encode(
+            &data,
+            &FilterContext {
+                filter: &good,
+                element_size: 2,
+                max_output: data.len(),
+            },
+        )
+        .unwrap();
+        for _ in 0..3_000 {
+            let cd: Vec<u32> = (0..rng.below(7))
+                .map(|_| match rng.below(4) {
+                    0 => rng.below(5) as u32,
+                    1 => u32::MAX - rng.below(4) as u32,
+                    2 => 1 << rng.below(32),
+                    _ => rng.next_u64() as u32,
+                })
+                .collect();
+            let f = ctx_for(cd);
+            let ctx = FilterContext {
+                filter: &f,
+                element_size: 2,
+                max_output: data.len(),
+            };
+            let _ = bitshuffle_decode(&enc, &ctx);
+            let _ = bitshuffle_decode(&data, &ctx);
+        }
+    }
+}

crates/clawhdf5-format/src/filters_blosc.rs

@@ -0,0 +1,711 @@
+//! Blosc 1 (HDF5 filter 32001, `hdf5-blosc`, hdf5plugin's `Blosc`), in pure
+//! Rust: the Blosc 1 frame, its byte shuffle and bit shuffle, and the
+//! BloscLZ, LZ4/LZ4HC, Snappy, Zlib and Zstandard codecs inside it.
+//!
+//! **Frame** (c-blosc 1.x, format version 2). A 16-byte header — version
+//! (2), codec format version (1), flags, type size, then little-endian `u32`
+//! decoded size, block size and frame size. Flags: bit 0 byte shuffle, bit
+//! 1 stored raw ("memcpyed": the data follows the header), bit 2 bit
+//! shuffle, bit 4 "do not split", bits 5-7 the codec (0 BloscLZ, 1 LZ4 and
+//! LZ4HC, 2 Snappy, 3 Zlib, 4 Zstandard). Unless stored raw, a table of
+//! `u32` block offsets follows, one per block of `block size` bytes (the
+//! last one may be shorter). A block is one stream, or — when the "do not
+//! split" flag is clear, the type size is at most 16, the block holds at
+//! least 128 elements, and it is not the short last block — `type size`
+//! streams, one per byte plane. Each stream is a `u32` length and the
+//! codec's output; a length equal to the stream's decoded size means the
+//! bytes are stored raw. The decoded block is then unshuffled (byte shuffle
+//! for type size > 1; bit shuffle when the block holds a multiple of 8
+//! elements, the trailing partial element copied as is).
+//!
+//! **Filter** (`blosc_filter.c`) `cd_values`: `[0]` filter revision, `[1]`
+//! Blosc format version, `[2]` type size, `[3]` chunk size in bytes, `[4]`
+//! compression level, `[5]` shuffle (0 none, 1 byte, 2 bit), `[6]`
+//! compressor (0 blosclz, 1 lz4, 2 lz4hc, 3 snappy, 4 zlib, 5 zstd). The
+//! decoder needs only the frame.
+
+use crate::error::FormatError;
+use crate::filter_registry::FilterContext;
+use crate::filters_bitshuffle::{bitshuffle_block, bitunshuffle_block};
+
+const HEADER: usize = 16;
+const FLAG_SHUFFLE: u8 = 0x01;
+const FLAG_MEMCPYED: u8 = 0x02;
+const FLAG_BITSHUFFLE: u8 = 0x04;
+const FLAG_FUTURE: u8 = 0x08;
+const FLAG_DONT_SPLIT: u8 = 0x10;
+const MAX_SPLITS: usize = 16;
+const MIN_BUFFERSIZE: usize = 128;
+
+fn err(msg: &str) -> FormatError {
+    FormatError::DecompressionError(format!("blosc: {msg}"))
+}
+
+fn le32(b: &[u8], at: usize) -> Result<usize, FormatError> {
+    b.get(at..at + 4)
+        .map(|s| u32::from_le_bytes(s.try_into().unwrap()) as usize)
+        .ok_or_else(|| err("truncated frame"))
+}
+
+/// The codec inside a Blosc frame (flags bits 5-7).
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum Codec {
+    BloscLz,
+    Lz4,
+    Snappy,
+    Zlib,
+    Zstd,
+}
+
+impl Codec {
+    fn from_flags(flags: u8) -> Result<Codec, FormatError> {
+        match flags >> 5 {
+            0 => Ok(Codec::BloscLz),
+            1 => Ok(Codec::Lz4),
+            2 => Ok(Codec::Snappy),
+            3 => Ok(Codec::Zlib),
+            4 => Ok(Codec::Zstd),
+            other => Err(err(&format!("unknown codec {other}"))),
+        }
+    }
+}
+
+/// Decode one codec stream into exactly `dst`.
+fn decode_stream(
+    codec: Codec,
+    src: &[u8],
+    dst: &mut [u8],
+    zstd: &mut Option<ruzstd::decoding::FrameDecoder>,
+) -> Result<(), FormatError> {
+    let n = match codec {
+        Codec::BloscLz => blosclz_decompress(src, dst),
+        Codec::Lz4 => {
+            lz4_flex::block::decompress_into(src, dst).map_err(|e| err(&format!("lz4: {e}")))?
+        }
+        Codec::Snappy => {
+            let len = snap::raw::decompress_len(src).map_err(|e| err(&format!("snappy: {e}")))?;
+            if len != dst.len() {
+                return Err(err("snappy stream has the wrong size"));
+            }
+            snap::raw::Decoder::new()
+                .decompress(src, dst)
+                .map_err(|e| err(&format!("snappy: {e}")))?
+        }
+        Codec::Zlib => {
+            let out = crate::filters::inflate_bounded(src, dst.len(), dst.len())
+                .map_err(|e| err(&format!("zlib: {e}")))?;
+            let n = out.len();
+            if n == dst.len() {
+                dst.copy_from_slice(&out);
+            }
+            n
+        }
+        Codec::Zstd => crate::filters_bitshuffle::zstd_decode_into(
+            zstd.get_or_insert_with(ruzstd::decoding::FrameDecoder::new),
+            src,
+            dst,
+        )?,
+    };
+    if n != dst.len() {
+        return Err(err("stream decoded to the wrong size"));
+    }
+    Ok(())
+}
+
+/// Decode a Blosc-filtered chunk: one Blosc 1 frame.
+///
+/// An HDF5 chunk is never empty, so a frame that decodes to nothing where
+/// the chunk size is known is corrupt (libhdf5's filter fails it too).
+pub(crate) fn blosc_decode(input: &[u8], ctx: &FilterContext<'_>) -> Result<Vec<u8>, FormatError> {
+    let out = blosc_decompress(input, ctx.output_limit())?;
+    if out.is_empty() && ctx.max_output != 0 {
+        return Err(err("empty frame for a non-empty chunk"));
+    }
+    Ok(out)
+}
+
+/// Decompress a Blosc 1 frame, refusing more than `limit` bytes of output.
+pub fn blosc_decompress(input: &[u8], limit: usize) -> Result<Vec<u8>, FormatError> {
+    if input.len() < HEADER {
+        return Err(err("truncated header"));
+    }
+    let version = input[0];
+    let codec_version = input[1];
+    let flags = input[2];
+    let typesize = input[3] as usize;
+    let nbytes = le32(input, 4)?;
+    let blocksize = le32(input, 8)?;
+    let cbytes = le32(input, 12)?;
+    if version != 1 && version != 2 {
+        return Err(err(&format!(
+            "frame format version {version} is not Blosc 1 (a Blosc 2 chunk?)"
+        )));
+    }
+    if flags & FLAG_FUTURE != 0 {
+        return Err(err("unknown header flags"));
+    }
+    if nbytes > limit {
+        return Err(err("decoded size exceeds the chunk size"));
+    }
+    if cbytes > input.len() {
+        return Err(err("frame is longer than the chunk"));
+    }
+    if cbytes < HEADER {
+        return Err(err("truncated frame"));
+    }
+    let src = &input[..cbytes];
+    if nbytes == 0 {
+        return Ok(Vec::new());
+    }
+    if blocksize == 0 || typesize == 0 {
+        return Err(err("bad block or type size"));
+    }
+    let mut out = vec![0u8; nbytes];
+    if flags & FLAG_MEMCPYED != 0 {
+        if cbytes != nbytes + HEADER {
+            return Err(err("stored frame has the wrong size"));
+        }
+        out.copy_from_slice(&src[HEADER..]);
+        return Ok(out);
+    }
+    let codec = Codec::from_flags(flags)?;
+    if codec_version != 1 {
+        return Err(err(&format!(
+            "unsupported {codec:?} format version {codec_version}"
+        )));
+    }
+    let nblocks = nbytes.div_ceil(blocksize);
+    let leftover = nbytes % blocksize;
+    if nblocks > (cbytes - HEADER) / 4 {
+        return Err(err("block table is truncated"));
+    }
+    let block_len = blocksize.min(nbytes);
+    let mut tmp = vec![0u8; block_len];
+    let mut zstd = None;
+    let dont_split = flags & FLAG_DONT_SPLIT != 0;
+    for j in 0..nblocks {
+        let is_leftover = j == nblocks - 1 && leftover > 0;
+        let bsize = if is_leftover { leftover } else { blocksize };
+        let nsplits = if !dont_split
+            && typesize <= MAX_SPLITS
+            && bsize / typesize >= MIN_BUFFERSIZE
+            && !is_leftover
+        {
+            typesize
+        } else {
+            1
+        };
+        let neblock = bsize / nsplits;
+        let mut pos = le32(src, HEADER + 4 * j)?;
+        let tmp = &mut tmp[..bsize];
+        for s in 0..nsplits {
+            let clen = src
+                .get(pos..)
+                .and_then(|rest| rest.get(..4))
+                .map(|b| u32::from_le_bytes(b.try_into().unwrap()) as usize)
+                .ok_or_else(|| err("block offset out of range"))?;
+            pos += 4;
+            let stream = src
+                .get(pos..pos.saturating_add(clen))
+                .ok_or_else(|| err("stream runs past the frame"))?;
+            let dst = &mut tmp[s * neblock..(s + 1) * neblock];
+            if clen == neblock {
+                dst.copy_from_slice(stream);
+            } else {
+                decode_stream(codec, stream, dst, &mut zstd)?;
+            }
+            pos += clen;
+        }
+        // `bsize` is a whole number of splits by construction (`nsplits` > 1
+        // only for full blocks, and c-blosc sizes those in whole elements).
+        if nsplits * neblock != bsize {
+            return Err(err("block is not a whole number of streams"));
+        }
+        let dest = &mut out[j * blocksize..j * blocksize + bsize];
+        unshuffle_block(flags, typesize, tmp, dest);
+    }
+    Ok(out)
+}
+
+/// Undo the frame's shuffle on one decoded block.
+fn unshuffle_block(flags: u8, typesize: usize, src: &[u8], dest: &mut [u8]) {
+    let bsize = src.len();
+    if flags & FLAG_SHUFFLE != 0 && typesize > 1 {
+        let n = bsize / typesize;
+        for i in 0..n {
+            for b in 0..typesize {
+                dest[i * typesize + b] = src[b * n + i];
+            }
+        }
+        dest[n * typesize..].copy_from_slice(&src[n * typesize..]);
+    } else if flags & FLAG_BITSHUFFLE != 0 && bsize >= typesize {
+        let n = bsize / typesize;
+        if n.is_multiple_of(8) {
+            let body = n * typesize;
+            bitunshuffle_block(&src[..body], &mut dest[..body], n, typesize);
+            dest[body..].copy_from_slice(&src[body..]);
+        } else {
+            dest.copy_from_slice(src);
+        }
+    } else {
+        dest.copy_from_slice(src);
+    }
+}
+
+/// BloscLZ decompression (c-blosc 1.21 `blosclz_decompress`): returns the
+/// number of bytes written, or 0 on malformed input — exactly as the C
+/// decoder, including stopping before a match that ends the stream, so a
+/// stream libblosc rejects is rejected here too.
+///
+/// Instructions: a control byte `ctrl`. Below 32, a literal run of
+/// `ctrl + 1` bytes. Otherwise a match: length `(ctrl >> 5) + 2`, extended
+/// by following bytes while they are 255 when the top three bits are all
+/// set; distance `((ctrl & 31) << 8) + next byte + 1`, or — when that byte
+/// is 255 and the high bits are 31 — a 16-bit big-endian distance plus 8192.
+/// The first instruction is always a literal.
+pub(crate) fn blosclz_decompress(input: &[u8], out: &mut [u8]) -> usize {
+    const MAX_DISTANCE: usize = 8191;
+    let limit = input.len();
+    if limit == 0 {
+        return 0;
+    }
+    let mut ip = 1usize;
+    let mut op = 0usize;
+    let mut ctrl = (input[0] & 31) as usize;
+    loop {
+        if ctrl >= 32 {
+            let mut len = (ctrl >> 5) - 1;
+            let ofs = (ctrl & 31) << 8;
+            if len == 6 {
+                loop {
+                    if ip + 1 >= limit {
+                        return 0;
+                    }
+                    let code = input[ip] as usize;
+                    ip += 1;
+                    len += code;
+                    if code != 255 {
+                        break;
+                    }
+                }
+            } else if ip + 1 >= limit {
+                return 0;
+            }
+            let code = input[ip] as usize;
+            ip += 1;
+            len += 3;
+            // The copy source is `distance` bytes back.
+            let mut distance = ofs + code + 1;
+            if code == 255 && ofs == 31 << 8 {
+                if ip + 1 >= limit {
+                    return 0;
+                }
+                let far = ((input[ip] as usize) << 8) + input[ip + 1] as usize;
+                ip += 2;
+                distance = far + MAX_DISTANCE + 1;
+            }
+            if op + len > out.len() {
+                return 0;
+            }
+            if distance > op {
+                return 0;
+            }
+            if ip >= limit {
+                break;
+            }
+            ctrl = input[ip] as usize;
+            ip += 1;
+            let start = op - distance;
+            if distance >= len {
+                out.copy_within(start..start + len, op);
+            } else {
+                for k in 0..len {
+                    out[op + k] = out[start + k];
+                }
+            }
+            op += len;
+        } else {
+            let run = ctrl + 1;
+            if op + run > out.len() || ip + run > limit {
+                return 0;
+            }
+            out[op..op + run].copy_from_slice(&input[ip..ip + run]);
+            op += run;
+            ip += run;
+            if ip >= limit {
+                break;
+            }
+            ctrl = input[ip] as usize;
+            ip += 1;
+        }
+    }
+    op
+}
+
+/// The codec our encoder puts inside the frame.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(crate) enum EncodeCodec {
+    Lz4,
+    Snappy,
+    Zlib,
+    Zstd,
+}
+
+impl EncodeCodec {
+    /// From the filter's `cd_values[6]` compressor code.
+    fn from_cd(code: u32) -> Result<EncodeCodec, FormatError> {
+        match code {
+            1 | 2 => Ok(EncodeCodec::Lz4),
+            3 => Ok(EncodeCodec::Snappy),
+            4 => Ok(EncodeCodec::Zlib),
+            5 => Ok(EncodeCodec::Zstd),
+            0 => Err(FormatError::CompressionError(
+                "blosc: clawhdf5 cannot write BloscLZ; choose lz4, snappy, zlib or zstd".into(),
+            )),
+            other => Err(FormatError::CompressionError(format!(
+                "blosc: unknown compressor {other}"
+            ))),
+        }
+    }
+
+    fn flags(self) -> u8 {
+        (match self {
+            EncodeCodec::Lz4 => 1,
+            EncodeCodec::Snappy => 2,
+            EncodeCodec::Zlib => 3,
+            EncodeCodec::Zstd => 4,
+        }) << 5
+    }
+
+    fn encode(self, data: &[u8], level: u32) -> Result<Vec<u8>, FormatError> {
+        match self {
+            EncodeCodec::Lz4 => Ok(lz4_flex::block::compress(data)),
+            EncodeCodec::Snappy => snap::raw::Encoder::new()
+                .compress_vec(data)
+                .map_err(|e| FormatError::CompressionError(format!("blosc: snappy: {e}"))),
+            EncodeCodec::Zlib => crate::filters::deflate_bounded(data, level.min(9))
+                .map_err(|e| FormatError::CompressionError(format!("blosc: zlib: {e}"))),
+            EncodeCodec::Zstd => Ok(crate::filters_bitshuffle::zstd_encode(data)),
+        }
+    }
+}
+
+/// Block size our encoder uses: at most 256 KiB, a whole number of
+/// elements (and, for bit shuffle, of 8-element groups).
+fn encode_block_size(nbytes: usize, typesize: usize, bitshuffle: bool) -> usize {
+    let unit = if bitshuffle { 8 * typesize } else { typesize };
+    let target = (256 * 1024).min(nbytes);
+    if target < unit {
+        return nbytes.max(1);
+    }
+    target / unit * unit
+}
+
+/// Encode a chunk as one Blosc 1 frame. `cd_values` as hdf5-blosc:
+/// `[2]` type size, `[4]` level (0 = store), `[5]` shuffle, `[6]` codec.
+pub(crate) fn blosc_encode(input: &[u8], ctx: &FilterContext<'_>) -> Result<Vec<u8>, FormatError> {
+    let cd = ctx.client_data();
+    let cerr = |m: &str| FormatError::CompressionError(format!("blosc: {m}"));
+    let typesize = match cd.get(2) {
+        Some(&t) if t != 0 => t as usize,
+        _ => ctx.element_size.max(1),
+    };
+    // Blosc records the type size in one byte; c-blosc treats larger types
+    // as bytes.
+    let typesize = if typesize > 255 { 1 } else { typesize };
+    let level = cd.get(4).copied().unwrap_or(5);
+    let shuffle = cd.get(5).copied().unwrap_or(1);
+    let codec = EncodeCodec::from_cd(cd.get(6).copied().unwrap_or(1))?;
+    let nbytes = input.len();
+    if nbytes > i32::MAX as usize - HEADER {
+        return Err(cerr("chunk too large for a Blosc frame"));
+    }
+    let mut flags = codec.flags();
+    match shuffle {
+        0 => {}
+        1 => flags |= FLAG_SHUFFLE,
+        2 => flags |= FLAG_BITSHUFFLE,
+        other => return Err(cerr(&format!("unknown shuffle mode {other}"))),
+    }
+    let blocksize = encode_block_size(nbytes, typesize, shuffle == 2);
+    let header = |flags: u8, blocksize: usize, cbytes: usize| {
+        let mut h = Vec::with_capacity(HEADER);
+        h.extend_from_slice(&[2, 1, flags, typesize as u8]);
+        h.extend_from_slice(&(nbytes as u32).to_le_bytes());
+        h.extend_from_slice(&(blocksize as u32).to_le_bytes());
+        h.extend_from_slice(&(cbytes as u32).to_le_bytes());
+        h
+    };
+    let stored = || {
+        let mut out = header(
+            FLAG_MEMCPYED | (flags & !(FLAG_SHUFFLE | FLAG_BITSHUFFLE)),
+            blocksize,
+            nbytes + HEADER,
+        );
+        out.extend_from_slice(input);
+        out
+    };
+    if level == 0 || nbytes == 0 {
+        return Ok(stored());
+    }
+
+    let nblocks = nbytes.div_ceil(blocksize);
+    let leftover = nbytes % blocksize;
+    let mut body = Vec::with_capacity(nbytes / 2);
+    let mut starts = Vec::with_capacity(nblocks);
+    let table_end = HEADER + 4 * nblocks;
+    let mut shuffled = vec![0u8; blocksize];
+    for j in 0..nblocks {
+        let is_leftover = j == nblocks - 1 && leftover > 0;
+        let bsize = if is_leftover { leftover } else { blocksize };
+        let block = &input[j * blocksize..j * blocksize + bsize];
+        let sh = &mut shuffled[..bsize];
+        shuffle_block(flags, typesize, block, sh);
+        starts.push(table_end + body.len());
+        let nsplits = if typesize <= MAX_SPLITS
+            && bsize / typesize >= MIN_BUFFERSIZE
+            && !is_leftover
+            && bsize.is_multiple_of(typesize)
+        {
+            typesize
+        } else {
+            1
+        };
+        let neblock = bsize / nsplits;
+        for s in 0..nsplits {
+            let part = &sh[s * neblock..(s + 1) * neblock];
+            let comp = codec.encode(part, level)?;
+            if comp.len() < neblock {
+                body.extend_from_slice(&(comp.len() as u32).to_le_bytes());
+                body.extend_from_slice(&comp);
+            } else {
+                body.extend_from_slice(&(neblock as u32).to_le_bytes());
+                body.extend_from_slice(part);
+            }
+        }
+        if table_end + body.len() >= nbytes + HEADER {
+            // Incompressible: store instead, as c-blosc does.
+            return Ok(stored());
+        }
+    }
+    // A split block must decode as split: the decoder infers splitting from
+    // the same rule, which requires a whole number of elements per block.
+    let cbytes = table_end + body.len();
+    let mut out = header(flags, blocksize, cbytes);
+    for s in starts {
+        out.extend_from_slice(&(s as u32).to_le_bytes());
+    }
+    out.extend_from_slice(&body);
+    Ok(out)
+}
+
+/// Apply the frame's shuffle to one block (the inverse of
+/// [`unshuffle_block`]).
+fn shuffle_block(flags: u8, typesize: usize, src: &[u8], dest: &mut [u8]) {
+    let bsize = src.len();
+    if flags & FLAG_SHUFFLE != 0 && typesize > 1 {
+        let n = bsize / typesize;
+        for i in 0..n {
+            for b in 0..typesize {
+                dest[b * n + i] = src[i * typesize + b];
+            }
+        }
+        dest[n * typesize..].copy_from_slice(&src[n * typesize..]);
+    } else if flags & FLAG_BITSHUFFLE != 0 && bsize >= typesize {
+        let n = bsize / typesize;
+        if n.is_multiple_of(8) {
+            let body = n * typesize;
+            bitshuffle_block(&src[..body], &mut dest[..body], n, typesize);
+            dest[body..].copy_from_slice(&src[body..]);
+        } else {
+            dest.copy_from_slice(src);
+        }
+    } else {
+        dest.copy_from_slice(src);
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::filter_pipeline::{FILTER_BLOSC, FilterDescription};
+
+    /// A blosclz stream: literal "abc", then a 9-byte match 3 back (a run
+    /// of "abc"), then literal "Z".
+    #[test]
+    fn blosclz_decodes_literals_and_overlapping_matches() {
+        // Match: length (ctrl >> 5) + 2 = 8, distance ofs + code + 1 = 3.
+        let stream = [2, b'a', b'b', b'c', (6 << 5), 2, 0, b'Z'];
+        let mut out = [0u8; 12];
+        assert_eq!(blosclz_decompress(&stream, &mut out), 12);
+        assert_eq!(&out, b"abcabcabcabZ");
+        // A stream cut inside a match is malformed.
+        let mut out = [0u8; 11];
+        assert_eq!(blosclz_decompress(&stream[..6], &mut out), 0);
+        // A match before the start of the output is malformed.
+        assert_eq!(blosclz_decompress(&[0, b'a', 32, 5, 0, b'x'], &mut out), 0);
+    }
+
+    fn desc(cd: Vec<u32>) -> FilterDescription {
+        FilterDescription {
+            filter_id: FILTER_BLOSC,
+            name: None,
+            flags: 0,
+            client_data: cd,
+        }
+    }
+
+    #[test]
+    fn frame_round_trips_every_codec_and_shuffle() {
+        for ts in [1usize, 2, 4, 8, 3, 32] {
+            for n in [0usize, 5, 100, 1000, 70_000, 300_001] {
+                if n * ts > 1 << 20 && ts > 1 {
+                    continue;
+                }
+                let data: Vec<u8> = (0..n * ts)
+                    .map(|i| ((i / ts) % 200) as u8 ^ (i % ts) as u8)
+                    .collect();
+                for codec in [1u32, 3, 4, 5] {
+                    for shuffle in [0u32, 1, 2] {
+                        for level in [0u32, 5] {
+                            let f = desc(vec![2, 2, ts as u32, 0, level, shuffle, codec]);
+                            let ctx = FilterContext {
+                                filter: &f,
+                                element_size: ts,
+                                max_output: data.len(),
+                            };
+                            let enc = blosc_encode(&data, &ctx).unwrap();
+                            let dec = blosc_decode(&enc, &ctx).unwrap_or_else(|e| {
+                                panic!("ts={ts} n={n} codec={codec} shuffle={shuffle}: {e}")
+                            });
+                            assert!(
+                                dec == data,
+                                "ts={ts} n={n} codec={codec} shuffle={shuffle} level={level}"
+                            );
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    #[test]
+    fn rejects_bad_frames() {
+        let data = vec![9u8; 50_000];
+        let f = desc(vec![2, 2, 4, 0, 5, 1, 1]);
+        let ctx = FilterContext {
+            filter: &f,
+            element_size: 4,
+            max_output: data.len(),
+        };
+        let enc = blosc_encode(&data, &ctx).unwrap();
+        assert!(blosc_decode(&enc[..enc.len() - 3], &ctx).is_err());
+        let small = FilterContext {
+            max_output: 49_999,
+            ..ctx
+        };
+        assert!(blosc_decode(&enc, &small).is_err());
+        let mut v3 = enc.clone();
+        v3[0] = 3;
+        assert!(blosc_decode(&v3, &ctx).is_err());
+        let f0 = desc(vec![2, 2, 4, 0, 5, 1, 0]);
+        let ctx0 = FilterContext { filter: &f0, ..ctx };
+        assert!(blosc_encode(&data, &ctx0).is_err());
+    }
+
+    /// A frame that declares no data, for a chunk that has some.
+    #[test]
+    fn empty_frame_for_a_non_empty_chunk_is_an_error() {
+        let mut frame = vec![2u8, 1, 0x20, 4];
+        for v in [0u32, 64, 16] {
+            frame.extend_from_slice(&v.to_le_bytes());
+        }
+        assert_eq!(blosc_decompress(&frame, 64).unwrap(), b"");
+        let f = desc(vec![2, 2, 4, 64, 5, 1, 1]);
+        let ctx = FilterContext {
+            filter: &f,
+            element_size: 4,
+            max_output: 64,
+        };
+        assert!(blosc_decode(&frame, &ctx).is_err());
+    }
+
+    /// A frame whose header claims a compressed size smaller than the
+    /// header itself, not stored raw: an error, not an arithmetic overflow
+    /// (it panicked in debug builds).
+    #[test]
+    fn frame_size_below_the_header_is_an_error() {
+        let mut frame = vec![2u8, 1, 1 << 5, 4];
+        for v in [64u32, 64, 8] {
+            frame.extend_from_slice(&v.to_le_bytes());
+        }
+        frame.extend_from_slice(&[0; 40]);
+        assert!(blosc_decompress(&frame, 1000).is_err());
+        for cbytes in 0..16u32 {
+            frame[12..16].copy_from_slice(&cbytes.to_le_bytes());
+            assert!(blosc_decompress(&frame, 1000).is_err(), "cbytes={cbytes}");
+        }
+    }
+
+    /// A BloscLZ frame (our encoder cannot write one): a single block,
+    /// one stream, no shuffle.
+    fn blosclz_frame() -> Vec<u8> {
+        let stream = [2, b'a', b'b', b'c', (6 << 5), 2, 0, b'Z'];
+        let mut f = vec![2u8, 1, 0, 1];
+        for v in [12u32, 12, (HEADER + 4 + 4 + stream.len()) as u32] {
+            f.extend_from_slice(&v.to_le_bytes());
+        }
+        f.extend_from_slice(&((HEADER + 4) as u32).to_le_bytes());
+        f.extend_from_slice(&(stream.len() as u32).to_le_bytes());
+        f.extend_from_slice(&stream);
+        f
+    }
+
+    /// Random and mutated frames, every codec and shuffle: errors are fine,
+    /// panics are not.
+    #[test]
+    fn fuzzed_frames_never_panic() {
+        let limit = 6000;
+        let data: Vec<u8> = (0..1500u32).flat_map(|i| (i / 5).to_le_bytes()).collect();
+        let mut seeds = vec![blosclz_frame()];
+        for codec in [1u32, 3, 4, 5] {
+            for shuffle in [0u32, 1, 2] {
+                for (ts, n) in [(4usize, data.len()), (4, 520), (1, 300), (2, 4)] {
+                    let f = desc(vec![2, 2, ts as u32, 0, 5, shuffle, codec]);
+                    let ctx = FilterContext {
+                        filter: &f,
+                        element_size: ts,
+                        max_output: n,
+                    };
+                    seeds.push(blosc_encode(&data[..n], &ctx).unwrap());
+                }
+            }
+        }
+        // Stored raw.
+        let f = desc(vec![2, 2, 4, 0, 0, 1, 1]);
+        let ctx = FilterContext {
+            filter: &f,
+            element_size: 4,
+            max_output: 64,
+        };
+        seeds.push(blosc_encode(&data[..64], &ctx).unwrap());
+        crate::test_fuzz::fuzz_decoder(0xb10, &seeds, 30_000, limit, |s| {
+            blosc_decompress(s, limit)
+        });
+    }
+
+    /// BloscLZ streams on their own, random and mutated.
+    #[test]
+    fn fuzzed_blosclz_streams_never_panic() {
+        let seed = blosclz_frame()[HEADER + 8..].to_vec();
+        let mut out = [0u8; 64];
+        crate::test_fuzz::fuzz_decoder(0xb11, &[seed], 30_000, 64, |s| {
+            let n = blosclz_decompress(s, &mut out);
+            if n == 0 {
+                Err(err("malformed"))
+            } else {
+                Ok(out[..n].to_vec())
+            }
+        });
+    }
+}

crates/clawhdf5-format/src/filters_bzip2.rs

@@ -0,0 +1,132 @@
+//! bzip2 (HDF5 filter 307, PyTables' `H5Zbzip2.c`, hdf5plugin's `BZip2`).
+//!
+//! The chunk is one bzip2 stream; `cd_values[0]` is the block size (1-9,
+//! the compression level). Decoded with the `bzip2` crate's default backend,
+//! `libbz2-rs-sys`, a pure-Rust port of libbzip2.
+
+use crate::error::FormatError;
+use crate::filter_registry::FilterContext;
+
+fn err(msg: &str) -> FormatError {
+    FormatError::DecompressionError(format!("bzip2: {msg}"))
+}
+
+/// Decode a bzip2-filtered chunk, refusing output beyond the chunk size.
+pub(crate) fn bzip2_decode(input: &[u8], ctx: &FilterContext<'_>) -> Result<Vec<u8>, FormatError> {
+    use bzip2::{Decompress, Status};
+    let limit = ctx.output_limit();
+    let max_capacity = limit.saturating_add(1);
+    let hint = if ctx.max_output != 0 {
+        ctx.max_output
+    } else {
+        input.len().saturating_mul(4)
+    };
+    let mut out = Vec::new();
+    out.try_reserve_exact(hint.clamp(1, max_capacity))
+        .map_err(|_| err("cannot allocate the output buffer"))?;
+    let mut dec = Decompress::new(false);
+    loop {
+        let (in_before, out_before) = (dec.total_in(), dec.total_out());
+        let status = dec
+            .decompress_vec(&input[in_before as usize..], &mut out)
+            .map_err(|e| err(&e.to_string()))?;
+        if out.len() > limit {
+            return Err(err("output exceeds the chunk size"));
+        }
+        if status == Status::StreamEnd {
+            return Ok(out);
+        }
+        if out.len() == out.capacity() {
+            let grow = out
+                .capacity()
+                .min(max_capacity.saturating_sub(out.capacity()))
+                .max(1);
+            out.try_reserve_exact(grow)
+                .map_err(|_| err("cannot allocate the output buffer"))?;
+        } else if dec.total_in() as usize >= input.len()
+            || (dec.total_in(), dec.total_out()) == (in_before, out_before)
+        {
+            return Err(err("truncated stream"));
+        }
+    }
+}
+
+/// Encode a chunk as one bzip2 stream at block size `cd_values[0]`
+/// (default 9, as hdf5plugin).
+pub(crate) fn bzip2_encode(input: &[u8], ctx: &FilterContext<'_>) -> Result<Vec<u8>, FormatError> {
+    use bzip2::{Action, Compress, Compression, Status};
+    let level = ctx.client_data().first().copied().unwrap_or(9).clamp(1, 9);
+    let cerr = |m: String| FormatError::CompressionError(format!("bzip2: {m}"));
+    let mut enc = Compress::new(Compression::new(level), 0);
+    // bzip2's worst case is about 1% + 600 bytes over the input.
+    let mut out = Vec::with_capacity(input.len() + input.len() / 100 + 600);
+    loop {
+        let consumed = enc.total_in() as usize;
+        let status = enc
+            .compress_vec(&input[consumed..], &mut out, Action::Finish)
+            .map_err(|e| cerr(e.to_string()))?;
+        if status == Status::StreamEnd {
+            return Ok(out);
+        }
+        if out.len() == out.capacity() {
+            out.reserve(out.capacity().max(4096));
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::filter_pipeline::{FILTER_BZIP2, FilterDescription};
+
+    fn desc(level: u32) -> FilterDescription {
+        FilterDescription {
+            filter_id: FILTER_BZIP2,
+            name: None,
+            flags: 0,
+            client_data: vec![level],
+        }
+    }
+
+    #[test]
+    fn round_trips_and_bounds() {
+        let data: Vec<u8> = (0..100_000u32)
+            .flat_map(|i| (i % 777).to_le_bytes())
+            .collect();
+        for level in [1, 5, 9] {
+            let f = desc(level);
+            let ctx = FilterContext {
+                filter: &f,
+                element_size: 4,
+                max_output: data.len(),
+            };
+            let enc = bzip2_encode(&data, &ctx).unwrap();
+            assert!(enc.len() < data.len() / 4);
+            assert_eq!(bzip2_decode(&enc, &ctx).unwrap(), data);
+            // Truncated, and larger than the chunk: errors, not data.
+            assert!(bzip2_decode(&enc[..enc.len() / 2], &ctx).is_err());
+            let small = FilterContext {
+                max_output: data.len() - 1,
+                ..ctx
+            };
+            assert!(bzip2_decode(&enc, &small).is_err());
+        }
+    }
+
+    /// Random and mutated streams: errors are fine, panics are not.
+    #[test]
+    fn fuzzed_streams_never_panic() {
+        let f = desc(9);
+        let data: Vec<u8> = (0..4000u32).flat_map(|i| (i % 91).to_le_bytes()).collect();
+        let ctx = FilterContext {
+            filter: &f,
+            element_size: 4,
+            max_output: data.len(),
+        };
+        let seeds = vec![
+            bzip2_encode(&data, &ctx).unwrap(),
+            bzip2_encode(&data[..40], &ctx).unwrap(),
+        ];
+        crate::test_fuzz::fuzz_decoder(0xb2, &seeds, 3_000, data.len(), |s| bzip2_decode(s, &ctx));
+    }
+}

crates/clawhdf5-format/src/filters_lzf.rs

@@ -0,0 +1,260 @@
+//! LZF (HDF5 filter 32000) — h5py's built-in compression filter
+//! (`compression="lzf"`), in pure Rust.
+//!
+//! The chunk is one raw LZF stream (liblzf 3.x format, no header). The
+//! stream is a sequence of instructions, each starting with a control byte:
+//!
+//! * `000LLLLL` — a literal run: the next `L + 1` bytes (1..=32) are copied.
+//! * `LLLOOOOO [E] OOOOOOOO` — a back reference: copy `len + 2` bytes from
+//!   `distance` bytes back, where `len` is the top three bits (1..=6), or
+//!   `7 + E` when they are all ones, and `distance` is the 13-bit offset
+//!   (high five bits in the control byte, low eight in the last byte) plus 1.
+//!
+//! h5py's filter (`lzf_filter.c`) records the chunk's size in bytes in
+//! `cd_values[2]` (slots 0 and 1 hold the filter and liblzf versions) and
+//! sizes its output buffer from it.
+
+#[cfg(not(feature = "std"))]
+extern crate alloc;
+#[cfg(not(feature = "std"))]
+use alloc::{format, vec, vec::Vec};
+
+use crate::error::FormatError;
+use crate::filter_registry::FilterContext;
+
+/// `H5PY_FILTER_LZF_VERSION`, written to `cd_values[0]`.
+pub const LZF_FILTER_VERSION: u32 = 4;
+/// `LZF_VERSION` (liblzf 1.5), written to `cd_values[1]`.
+pub const LZF_API_VERSION: u32 = 0x0105;
+
+const MAX_LITERAL: usize = 32;
+const MAX_OFFSET: usize = 1 << 13;
+const MAX_REF: usize = (1 << 8) + (1 << 3);
+const HASH_LOG: u32 = 14;
+
+fn err(msg: &str) -> FormatError {
+    FormatError::DecompressionError(format!("lzf: {msg}"))
+}
+
+/// Decode an LZF-filtered chunk.
+pub(crate) fn lzf_decode(input: &[u8], ctx: &FilterContext<'_>) -> Result<Vec<u8>, FormatError> {
+    let limit = ctx.output_limit();
+    let hint = match ctx.client_data().get(2) {
+        Some(&n) if n != 0 => n as usize,
+        _ => input.len().saturating_mul(2),
+    };
+    lzf_decompress(input, hint.min(limit), limit)
+}
+
+/// Decompress a raw LZF stream, refusing to produce more than `limit` bytes.
+pub fn lzf_decompress(
+    input: &[u8],
+    size_hint: usize,
+    limit: usize,
+) -> Result<Vec<u8>, FormatError> {
+    let mut out: Vec<u8> = Vec::new();
+    out.try_reserve(size_hint)
+        .map_err(|_| err("cannot allocate the output buffer"))?;
+    let mut ip = 0usize;
+    while ip < input.len() {
+        let ctrl = input[ip] as usize;
+        ip += 1;
+        if ctrl < 32 {
+            let run = ctrl + 1;
+            let lit = input
+                .get(ip..ip + run)
+                .ok_or_else(|| err("literal run past the end of the input"))?;
+            if out.len() + run > limit {
+                return Err(err("output exceeds the chunk size"));
+            }
+            out.extend_from_slice(lit);
+            ip += run;
+        } else {
+            let mut len = ctrl >> 5;
+            if len == 7 {
+                len += *input
+                    .get(ip)
+                    .ok_or_else(|| err("truncated back reference"))?
+                    as usize;
+                ip += 1;
+            }
+            let low = *input
+                .get(ip)
+                .ok_or_else(|| err("truncated back reference"))? as usize;
+            ip += 1;
+            let distance = ((ctrl & 0x1f) << 8) + low + 1;
+            let len = len + 2;
+            if distance > out.len() {
+                return Err(err("back reference before the start of the output"));
+            }
+            if out.len() + len > limit {
+                return Err(err("output exceeds the chunk size"));
+            }
+            let start = out.len() - distance;
+            if distance >= len {
+                out.extend_from_within(start..start + len);
+            } else {
+                // Overlapping copy: repeats the last `distance` bytes.
+                for k in 0..len {
+                    let b = out[start + k];
+                    out.push(b);
+                }
+            }
+        }
+    }
+    Ok(out)
+}
+
+/// Encode a chunk with the LZF filter.
+pub(crate) fn lzf_encode(input: &[u8], _ctx: &FilterContext<'_>) -> Result<Vec<u8>, FormatError> {
+    Ok(lzf_compress(input))
+}
+
+fn hash3(b: &[u8]) -> usize {
+    let v = (u32::from(b[0]) << 16) | (u32::from(b[1]) << 8) | u32::from(b[2]);
+    (v.wrapping_mul(2_654_435_761) >> (32 - HASH_LOG)) as usize
+}
+
+fn flush_literals(out: &mut Vec<u8>, lit: &[u8]) {
+    for run in lit.chunks(MAX_LITERAL) {
+        out.push((run.len() - 1) as u8);
+        out.extend_from_slice(run);
+    }
+}
+
+/// Compress `input` into a raw LZF stream any liblzf decoder reads.
+///
+/// Incompressible input grows by one byte per 32. (h5py's own filter gives
+/// up on such a chunk and stores it unfiltered; storing the slightly larger
+/// stream is equally readable.)
+pub fn lzf_compress(input: &[u8]) -> Vec<u8> {
+    let n = input.len();
+    let mut out = Vec::with_capacity(n + n / MAX_LITERAL + 1);
+    let mut table = vec![0u32; 1 << HASH_LOG];
+    let mut lit_start = 0usize;
+    let mut i = 0usize;
+    while i + 2 < n {
+        let h = hash3(&input[i..]);
+        let cand = table[h] as usize;
+        table[h] = (i + 1) as u32;
+        if cand != 0 {
+            let r = cand - 1;
+            let distance = i - r;
+            if distance <= MAX_OFFSET && input[r..r + 3] == input[i..i + 3] {
+                let max_len = (n - i).min(MAX_REF);
+                let mut len = 3;
+                while len < max_len && input[r + len] == input[i + len] {
+                    len += 1;
+                }
+                flush_literals(&mut out, &input[lit_start..i]);
+                let code = len - 2;
+                let off = distance - 1;
+                if code < 7 {
+                    out.push(((code << 5) | (off >> 8)) as u8);
+                } else {
+                    out.push(((7 << 5) | (off >> 8)) as u8);
+                    out.push((code - 7) as u8);
+                }
+                out.push((off & 0xff) as u8);
+                // Index the positions the match covered so later data can
+                // refer back into it.
+                let end = i + len;
+                let mut j = i + 1;
+                while j < end && j + 2 < n {
+                    table[hash3(&input[j..])] = (j + 1) as u32;
+                    j += 1;
+                }
+                i = end;
+                lit_start = i;
+                continue;
+            }
+        }
+        i += 1;
+    }
+    flush_literals(&mut out, &input[lit_start..]);
+    out
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn round_trip(data: &[u8]) {
+        let c = lzf_compress(data);
+        assert_eq!(lzf_decompress(&c, data.len(), data.len()).unwrap(), data);
+    }
+
+    #[test]
+    fn round_trips() {
+        round_trip(b"");
+        round_trip(b"a");
+        round_trip(b"abcabcabcabcabcabcabcabcabcabcabcabc");
+        round_trip(&[7u8; 10_000]);
+        let noise: Vec<u8> = (0..70_000u32)
+            .map(|i| (i.wrapping_mul(2_654_435_761) >> 13) as u8)
+            .collect();
+        round_trip(&noise);
+        let ramp: Vec<u8> = (0..100_000u32)
+            .flat_map(|i| (i % 1000).to_le_bytes())
+            .collect();
+        round_trip(&ramp);
+    }
+
+    #[test]
+    fn compresses_repetitive_data() {
+        let data = [42u8; 4096];
+        assert!(lzf_compress(&data).len() < 100);
+    }
+
+    /// The chunk h5py 3.16's bundled liblzf writes for
+    /// `b"hello hello hello hello"` (read back with `read_direct_chunk`): a
+    /// 7-byte literal, a 14-byte back reference 6 bytes back (extended
+    /// length), and a 2-byte literal.
+    #[test]
+    fn decodes_liblzf_output() {
+        let stream = b"\x06hello h\xe0\x05\x05\x01lo";
+        assert_eq!(
+            lzf_decompress(stream, 23, 23).unwrap(),
+            b"hello hello hello hello"
+        );
+    }
+
+    #[test]
+    fn rejects_corrupt_streams() {
+        // Back reference before the start.
+        assert!(lzf_decompress(&[0x20, 0x00], 10, 10).is_err());
+        // Literal run past the end.
+        assert!(lzf_decompress(&[0x05, 1, 2], 10, 10).is_err());
+        // Output over the limit.
+        let c = lzf_compress(&[1u8; 100]);
+        assert!(lzf_decompress(&c, 10, 99).is_err());
+    }
+
+    /// Random and mutated streams: errors are fine, panics are not.
+    #[test]
+    fn fuzzed_streams_never_panic() {
+        let seeds: Vec<Vec<u8>> = [
+            b"hello hello hello hello".to_vec(),
+            vec![7u8; 3000],
+            (0..2000u32).flat_map(|i| (i % 37).to_le_bytes()).collect(),
+            (0..500u32)
+                .map(|i| (i.wrapping_mul(2_654_435_761) >> 13) as u8)
+                .collect(),
+        ]
+        .iter()
+        .map(|d| lzf_compress(d))
+        .collect();
+        for limit in [0usize, 23, 4096, 8000] {
+            crate::test_fuzz::fuzz_decoder(
+                0x1f2 + limit as u64,
+                &seeds[..1],
+                5_000,
+                limit.max(23),
+                |s| lzf_decompress(s, limit, limit.max(23)),
+            );
+        }
+        crate::test_fuzz::fuzz_decoder(0x1f3, &seeds, 20_000, 8000, |s| {
+            lzf_decompress(s, 8000, 8000)
+        });
+    }
+}

crates/clawhdf5-format/src/filters_szip.rs

@@ -34,6 +34,7 @@ const SZ_NN_OPTION_MASK: u32 = 32;
 ///
 /// The chunk is a 4-byte little-endian uncompressed size followed by the
 /// szlib stream.
+#[cfg_attr(not(feature = "szip"), allow(dead_code))]
 pub(crate) fn szip_decompress(
     _data: &[u8],
     _cd: &[u32],

crates/clawhdf5-format/src/fractal_heap.rs

@@ -1,12 +1,14 @@
 //! HDF5 Fractal Heap parsing for v2 group link storage.
 
 #[cfg(not(feature = "std"))]
-use alloc::vec::Vec;
+use alloc::{format, vec::Vec};
 
 #[cfg(feature = "checksum")]
 use byteorder::{ByteOrder, LittleEndian};
 
+use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records};
 use crate::error::FormatError;
+use crate::filter_pipeline::FilterPipeline;
 
 /// Parsed fractal heap header (signature "FRHP").
 #[derive(Debug, Clone)]
@@ -33,6 +35,23 @@ pub struct FractalHeapHeader {
     pub current_rows_in_root_indirect_block: u16,
     /// Total number of managed objects.
     pub managed_objects_count: u64,
+    /// Address of the v2 B-tree indexing "huge" objects (undefined address
+    /// when the heap has none). Huge objects are those larger than
+    /// `max_managed_object_size`; they live outside the heap's blocks.
+    pub huge_btree_address: u64,
+    /// The heap's I/O filter pipeline, if it has one. It applies to managed
+    /// direct blocks and to huge objects.
+    pub filter_pipeline: Option<FilterPipeline>,
+    /// Stored (filtered) size of the root direct block; meaningful only when
+    /// the heap is filtered and its root is a direct block.
+    pub root_direct_block_filtered_size: u64,
+    /// Filter mask of the root direct block (bit *i* set = filter *i*
+    /// skipped); meaningful only when the heap is filtered.
+    pub root_direct_block_filter_mask: u32,
+    /// Size of addresses in the file ("Size of Offsets").
+    pub offset_size: u8,
+    /// Size of lengths in the file ("Size of Lengths").
+    pub length_size: u8,
 }
 
 fn read_offset(data: &[u8], pos: usize, size: u8) -> Result<u64, FormatError> {
@@ -79,6 +98,38 @@ fn is_undefined(val: u64, offset_size: u8) -> bool {
     }
 }
 
+/// Little-endian unsigned integer of up to 8 bytes.
+fn le_uint(bytes: &[u8]) -> u64 {
+    bytes
+        .iter()
+        .take(8)
+        .enumerate()
+        .fold(0u64, |acc, (i, &b)| acc | (u64::from(b) << (i * 8)))
+}
+
+fn heap_error(msg: &str) -> FormatError {
+    FormatError::ChunkedReadError(format!("fractal heap: {msg}"))
+}
+
+/// Heap ID type, from bits 4-5 of an ID's first byte (libhdf5's
+/// `H5HF_ID_TYPE_MASK`, 0x30); bits 6-7 are the ID version, which must be 0.
+const HEAP_ID_MANAGED: u8 = 0;
+const HEAP_ID_HUGE: u8 = 1;
+const HEAP_ID_TINY: u8 = 2;
+
+/// The type (0 managed, 1 huge, 2 tiny) of a heap ID from its first byte,
+/// refusing an ID version other than 0.
+fn heap_id_type(first: u8) -> Result<u8, FormatError> {
+    if first >> 6 != 0 {
+        return Err(heap_error("unsupported heap ID version"));
+    }
+    Ok((first >> 4) & 0x03)
+}
+
+/// v2 B-tree record types indexing a heap's huge objects.
+const BTREE_HUGE_INDIRECT: u8 = 1;
+const BTREE_HUGE_INDIRECT_FILTERED: u8 = 2;
+
 impl FractalHeapHeader {
     /// Parse a fractal heap header at the given offset.
     pub fn parse(
@@ -122,11 +173,17 @@ impl FractalHeapHeader {
         ]);
         pos += 4;
 
-        // Skip several fixed fields: next_huge_object_id(ls), btree_huge_objects_address(os),
+        // next_huge_object_id (length_size)
-        // free_space_managed_blocks(ls), managed_block_free_space_manager_address(os),
+        ensure_len(file_data, pos, ls)?;
+        pos += ls;
+        // btree_huge_objects_address (offset_size)
+        let huge_btree_address = read_offset(file_data, pos, offset_size)?;
+        pos += os;
+
+        // Skip: free_space_managed_blocks(ls), managed_block_free_space_manager_address(os),
         // managed_space_in_heap(ls), allocated_managed_space_in_heap(ls),
         // direct_block_allocation_iterator_offset(ls)
-        let skip_size = 5 * ls + 2 * os;
+        let skip_size = 4 * ls + os;
         ensure_len(file_data, pos, skip_size)?;
         pos += skip_size;
 
@@ -134,14 +191,9 @@ impl FractalHeapHeader {
         let managed_objects_count = read_offset(file_data, pos, length_size)?;
         pos += ls;
 
-        // huge_objects_size (length_size)
+        // huge_objects_size, huge_objects_count, tiny_objects_size,
-        pos += ls;
+        // tiny_objects_count (length_size each)
-        // huge_objects_count (length_size)
+        pos += 4 * ls;
-        pos += ls;
-        // tiny_objects_size (length_size)
-        pos += ls;
-        // tiny_objects_count (length_size)
-        pos += ls;
 
         // table_width (2)
         ensure_len(file_data, pos, 2)?;
@@ -175,16 +227,28 @@ impl FractalHeapHeader {
         ensure_len(file_data, pos, 2)?;
         let current_rows_in_root_indirect_block =
             u16::from_le_bytes([file_data[pos], file_data[pos + 1]]);
-        #[allow(unused_variables, unused_mut, unused_assignments)]
+        pos += 2;
-        let mut pos = pos + 2;
 
-        // Skip IO filter encoded info if present
+        // With I/O filters: root direct block's filtered size (length_size),
+        // its filter mask (4), then the encoded filter pipeline message.
+        let mut filter_pipeline = None;
+        let mut root_direct_block_filtered_size = 0;
+        let mut root_direct_block_filter_mask = 0;
         if io_filter_encoded_length > 0 {
-            // root_block_filter_info_size (length_size) + filter_mask (4)
+            root_direct_block_filtered_size = read_offset(file_data, pos, length_size)?;
-            #[allow(unused_assignments)]
+            pos += ls;
-            {
+            ensure_len(file_data, pos, 4)?;
-                pos += ls + 4;
+            root_direct_block_filter_mask = u32::from_le_bytes([
-            }
+                file_data[pos],
+                file_data[pos + 1],
+                file_data[pos + 2],
+                file_data[pos + 3],
+            ]);
+            pos += 4;
+            let n = io_filter_encoded_length as usize;
+            ensure_len(file_data, pos, n)?;
+            filter_pipeline = Some(FilterPipeline::parse(&file_data[pos..pos + n])?);
+            pos += n;
         }
 
         // Validate header checksum
@@ -200,6 +264,8 @@ impl FractalHeapHeader {
                 });
             }
         }
+        #[cfg(not(feature = "checksum"))]
+        let _ = pos;
 
         Ok(FractalHeapHeader {
             heap_id_length,
@@ -213,13 +279,19 @@ impl FractalHeapHeader {
             root_block_address,
             current_rows_in_root_indirect_block,
             managed_objects_count,
+            huge_btree_address,
+            filter_pipeline,
+            root_direct_block_filtered_size,
+            root_direct_block_filter_mask,
+            offset_size,
+            length_size,
         })
     }
 
     /// Decode a managed heap ID into (offset_in_heap, object_length).
     ///
     /// The heap ID layout for managed objects (type 0):
-    /// - Byte 0: bits 6-7 = type (0), bits 4-5 = version (0), bits 0-3 = reserved
+    /// - Byte 0: bits 6-7 = version (0), bits 4-5 = type (0), bits 0-3 = reserved
     /// - Bytes 1+: offset (max_heap_size bits, LE) then length (remaining bits, LE)
     pub fn decode_managed_id(&self, id_bytes: &[u8]) -> Result<(u64, u64), FormatError> {
         if id_bytes.is_empty() {
@@ -229,8 +301,8 @@ impl FractalHeapHeader {
             });
         }
 
-        let id_type = (id_bytes[0] >> 6) & 0x03;
+        let id_type = heap_id_type(id_bytes[0])?;
-        if id_type != 0 {
+        if id_type != HEAP_ID_MANAGED {
             return Err(FormatError::InvalidHeapIdType(id_type));
         }
 
@@ -269,12 +341,183 @@ impl FractalHeapHeader {
         Ok((heap_offset, length_val))
     }
 
-    /// Read a managed object from the heap given its raw heap ID bytes.
+    /// Read any object from the heap given its raw heap ID bytes: managed
+    /// (stored in the heap's blocks), huge (stored outside them, found
+    /// directly from the ID or through the huge-object v2 B-tree, optionally
+    /// filtered) or tiny (stored in the ID itself).
+    ///
+    /// Despite its name this accepts every ID type; `offset_size` must match
+    /// the one the header was parsed with.
     pub fn read_managed_object(
         &self,
         file_data: &[u8],
         id_bytes: &[u8],
         offset_size: u8,
+    ) -> Result<Vec<u8>, FormatError> {
+        let Some(&first) = id_bytes.first() else {
+            return Err(FormatError::UnexpectedEof {
+                expected: 1,
+                available: 0,
+            });
+        };
+        match heap_id_type(first)? {
+            HEAP_ID_MANAGED => self.read_heap_managed(file_data, id_bytes, offset_size),
+            HEAP_ID_HUGE => self.read_huge_object(file_data, id_bytes),
+            HEAP_ID_TINY => self.read_tiny_object(id_bytes),
+            other => Err(FormatError::InvalidHeapIdType(other)),
+        }
+    }
+
+    /// Whether a huge object's ID holds its address and length directly
+    /// (libhdf5 does this when they fit in the ID), rather than a key into
+    /// the huge-object B-tree.
+    fn huge_ids_direct(&self) -> bool {
+        let room = usize::from(self.heap_id_length).saturating_sub(1);
+        let os = usize::from(self.offset_size);
+        let ls = usize::from(self.length_size);
+        if self.filter_pipeline.is_some() {
+            room >= os + ls + 4 + ls
+        } else {
+            room >= os + ls
+        }
+    }
+
+    /// Read a huge object (heap ID type 1).
+    fn read_huge_object(&self, file_data: &[u8], id: &[u8]) -> Result<Vec<u8>, FormatError> {
+        let os = usize::from(self.offset_size);
+        let ls = usize::from(self.length_size);
+        // (address, stored length, filter mask, decoded length); the last two
+        // only matter for a filtered heap.
+        let (addr, stored_len, mask, mem_len) = if self.huge_ids_direct() {
+            let body = &id[1..];
+            let need = if self.filter_pipeline.is_some() {
+                os + ls + 4 + ls
+            } else {
+                os + ls
+            };
+            ensure_len(body, 0, need)?;
+            let addr = le_uint(&body[..os]);
+            let len = le_uint(&body[os..os + ls]);
+            if self.filter_pipeline.is_some() {
+                let mask = u32::from_le_bytes([
+                    body[os + ls],
+                    body[os + ls + 1],
+                    body[os + ls + 2],
+                    body[os + ls + 3],
+                ]);
+                let mem = le_uint(&body[os + ls + 4..os + ls + 4 + ls]);
+                (addr, len, mask, mem)
+            } else {
+                (addr, len, 0, len)
+            }
+        } else {
+            let key_len = (usize::from(self.heap_id_length).saturating_sub(1)).min(8);
+            ensure_len(id, 1, key_len)?;
+            let key = le_uint(&id[1..1 + key_len]);
+            self.find_huge_record(file_data, key)?
+        };
+
+        let start = usize::try_from(addr).map_err(|_| heap_error("huge object address"))?;
+        let len = usize::try_from(stored_len).map_err(|_| heap_error("huge object length"))?;
+        ensure_len(file_data, start, len)?;
+        let stored = &file_data[start..start + len];
+        match &self.filter_pipeline {
+            None => Ok(stored.to_vec()),
+            Some(pipeline) => {
+                let mem = usize::try_from(mem_len).map_err(|_| heap_error("huge object size"))?;
+                let out = crate::filters::decompress_chunk_masked(stored, pipeline, mem, 1, mask)?;
+                if out.len() != mem {
+                    return Err(heap_error("filtered huge object decoded to the wrong size"));
+                }
+                Ok(out)
+            }
+        }
+    }
+
+    /// Look up huge object `key` in the huge-object v2 B-tree, returning
+    /// (address, stored length, filter mask, decoded length).
+    fn find_huge_record(
+        &self,
+        file_data: &[u8],
+        key: u64,
+    ) -> Result<(u64, u64, u32, u64), FormatError> {
+        if is_undefined(self.huge_btree_address, self.offset_size) {
+            return Err(heap_error(
+                "huge object ID but the heap has no huge-object index",
+            ));
+        }
+        let hdr = BTreeV2Header::parse(
+            file_data,
+            self.huge_btree_address as usize,
+            self.offset_size,
+            self.length_size,
+        )?;
+        let os = usize::from(self.offset_size);
+        let ls = usize::from(self.length_size);
+        let filtered = self.filter_pipeline.is_some();
+        let (expected_type, rec_len) = if filtered {
+            (BTREE_HUGE_INDIRECT_FILTERED, os + ls + 4 + ls + ls)
+        } else {
+            (BTREE_HUGE_INDIRECT, os + ls + ls)
+        };
+        if hdr.tree_type != expected_type || usize::from(hdr.record_size) < rec_len {
+            return Err(heap_error("unexpected huge-object B-tree record type"));
+        }
+        let records =
+            collect_btree_v2_records(file_data, &hdr, self.offset_size, self.length_size)?;
+        for rec in &records {
+            let d = &rec.data;
+            if d.len() < rec_len {
+                continue;
+            }
+            let addr = le_uint(&d[..os]);
+            let len = le_uint(&d[os..os + ls]);
+            if filtered {
+                let mask = u32::from_le_bytes([
+                    d[os + ls],
+                    d[os + ls + 1],
+                    d[os + ls + 2],
+                    d[os + ls + 3],
+                ]);
+                let mem = le_uint(&d[os + ls + 4..os + 2 * ls + 4]);
+                let id = le_uint(&d[os + 2 * ls + 4..os + 3 * ls + 4]);
+                if id == key {
+                    return Ok((addr, len, mask, mem));
+                }
+            } else {
+                let id = le_uint(&d[os + ls..os + 2 * ls]);
+                if id == key {
+                    return Ok((addr, len, 0, len));
+                }
+            }
+        }
+        Err(heap_error("huge object not found in its B-tree"))
+    }
+
+    /// Read a tiny object (heap ID type 2), stored in the ID itself.
+    fn read_tiny_object(&self, id: &[u8]) -> Result<Vec<u8>, FormatError> {
+        // libhdf5 uses a one-byte length (low 4 bits of byte 0) unless the ID
+        // is long enough to need 12 bits, which then borrow byte 1.
+        let extended = usize::from(self.heap_id_length).saturating_sub(1) > 17;
+        let (len, start) = if extended {
+            ensure_len(id, 0, 2)?;
+            (
+                ((usize::from(id[0] & 0x0F)) << 8 | usize::from(id[1])) + 1,
+                2,
+            )
+        } else {
+            (usize::from(id[0] & 0x0F) + 1, 1)
+        };
+        ensure_len(id, start, len)?;
+        Ok(id[start..start + len].to_vec())
+    }
+
+    /// Read a managed object (heap ID type 0).
+    fn read_heap_managed(
+        &self,
+        file_data: &[u8],
+        id_bytes: &[u8],
+        offset_size: u8,
     ) -> Result<Vec<u8>, FormatError> {
         let (heap_offset, obj_len) = self.decode_managed_id(id_bytes)?;
 
@@ -289,12 +532,15 @@ impl FractalHeapHeader {
             // Root is a direct block
             self.read_from_direct_block(
                 file_data,
-                self.root_block_address as usize,
+                DirectBlock {
-                self.starting_block_size,
+                    addr: self.root_block_address as usize,
-                0, // block offset in heap = 0 for root
+                    size: self.starting_block_size,
+                    heap_offset: 0,
+                    filtered_size: self.root_direct_block_filtered_size,
+                    filter_mask: self.root_direct_block_filter_mask,
+                },
                 heap_offset,
                 obj_len as usize,
-                offset_size,
             )
         } else {
             // Root is an indirect block — limit recursion to 64 levels
@@ -313,27 +559,41 @@ impl FractalHeapHeader {
 
     /// Read an object from a direct block.
     ///
-    /// The heap offset is relative to the start of the block (including its header),
+    /// The heap offset is relative to the start of the block (including its
-    /// so we just add it to the block address minus the block's heap offset.
+    /// header), so we just add it to the block address minus the block's heap
-    #[allow(clippy::too_many_arguments)]
+    /// offset. A filtered heap stores each direct block (header included)
+    /// through its filter pipeline, so the block is decoded first.
     fn read_from_direct_block(
         &self,
         file_data: &[u8],
-        block_addr: usize,
+        block: DirectBlock,
-        _block_size: u64,
-        block_heap_offset: u64,
         target_offset: u64,
         length: usize,
-        _offset_size: u8,
     ) -> Result<Vec<u8>, FormatError> {
-        if target_offset < block_heap_offset {
+        if target_offset < block.heap_offset {
             return Err(FormatError::UnexpectedEof {
-                expected: block_heap_offset as usize,
+                expected: block.heap_offset as usize,
                 available: target_offset as usize,
             });
         }
-        let local_offset = (target_offset - block_heap_offset) as usize;
+        let local_offset = (target_offset - block.heap_offset) as usize;
-        let pos = block_addr
+        if let Some(pipeline) = &self.filter_pipeline {
+            let stored_len = usize::try_from(block.filtered_size)
+                .map_err(|_| heap_error("direct block size"))?;
+            let size = usize::try_from(block.size).map_err(|_| heap_error("direct block size"))?;
+            ensure_len(file_data, block.addr, stored_len)?;
+            let decoded = crate::filters::decompress_chunk_masked(
+                &file_data[block.addr..block.addr + stored_len],
+                pipeline,
+                size,
+                1,
+                block.filter_mask,
+            )?;
+            ensure_len(&decoded, local_offset, length)?;
+            return Ok(decoded[local_offset..local_offset + length].to_vec());
+        }
+        let pos = block
+            .addr
             .checked_add(local_offset)
             .ok_or(FormatError::UnexpectedEof {
                 expected: usize::MAX,
@@ -371,19 +631,13 @@ impl FractalHeapHeader {
         let iblock_header = 5 + offset_size as usize + block_offset_bytes;
         let mut pos = iblock_addr + iblock_header;
 
-        // Compute block sizes for each row using the doubling table
         let tw = self.table_width as u64;
-
         let nrows_usize = nrows as usize;
-
-        // Build table of (block_size, heap_offset) for each child entry
         let mut current_heap_offset = iblock_heap_offset;
 
         // Rows below max_direct_rows hold direct blocks; rows at/above hold
         // child indirect blocks. (NOT the FRHP "starting rows" field.)
         let start_indirect = self.max_direct_rows();
-
-        // Read child addresses for direct block rows
         let max_direct_rows = nrows_usize.min(start_indirect);
 
         for row in 0..max_direct_rows {
@@ -393,48 +647,66 @@ impl FractalHeapHeader {
                 let child_addr = read_offset(file_data, pos, offset_size)?;
                 pos += offset_size as usize;
 
-                if self.io_filter_encoded_length > 0 {
+                // A filtered heap stores each direct block's filtered size
-                    // filtered_size(length_size) + filter_mask(4)
+                // (length_size) and filter mask (4) after its address.
-                    // Skip for now - we don't handle filtered direct blocks in fractal heaps
+                let (filtered_size, filter_mask) = if self.filter_pipeline.is_some() {
-                    pos += 4; // filter_mask - simplified
+                    let size = read_offset(file_data, pos, self.length_size)?;
-                }
+                    pos += usize::from(self.length_size);
+                    ensure_len(file_data, pos, 4)?;
+                    let mask = u32::from_le_bytes([
+                        file_data[pos],
+                        file_data[pos + 1],
+                        file_data[pos + 2],
+                        file_data[pos + 3],
+                    ]);
+                    pos += 4;
+                    (size, mask)
+                } else {
+                    (0, 0)
+                };
 
-                if !is_undefined(child_addr, offset_size) {
+                let block_end = current_heap_offset.saturating_add(block_size);
-                    let block_end = current_heap_offset + block_size;
+                if !is_undefined(child_addr, offset_size)
-                    if target_offset >= current_heap_offset && target_offset < block_end {
+                    && target_offset >= current_heap_offset
+                    && target_offset < block_end
+                {
                     return self.read_from_direct_block(
                         file_data,
-                            child_addr as usize,
+                        DirectBlock {
-                            block_size,
+                            addr: child_addr as usize,
-                            current_heap_offset,
+                            size: block_size,
+                            heap_offset: current_heap_offset,
+                            filtered_size,
+                            filter_mask,
+                        },
                         target_offset,
                         length,
-                            offset_size,
                     );
                 }
-                }
+                current_heap_offset = block_end;
-                current_heap_offset += block_size;
             }
         }
 
-        // If we have indirect block rows
+        // Rows at and above `start_indirect` hold child indirect blocks. A
+        // child in row r spans exactly that row's block size of heap space,
+        // so it has as many rows as a table of that total size needs.
         for row in start_indirect..nrows_usize {
-            let _block_size = self.block_size_for_row(row);
+            let child_space = self.block_size_for_row(row);
-            let child_nrows = row - start_indirect + 1;
+            let child_nrows = self.rows_for_size(child_space);
 
             for _col in 0..tw {
                 let child_addr = read_offset(file_data, pos, offset_size)?;
                 pos += offset_size as usize;
 
-                if !is_undefined(child_addr, offset_size) {
+                let block_end = current_heap_offset.saturating_add(child_space);
-                    // Calculate total heap space covered by this indirect block child
+                if !is_undefined(child_addr, offset_size)
-                    let total_child_space = self.indirect_block_heap_size(child_nrows);
+                    && target_offset >= current_heap_offset
-                    let block_end = current_heap_offset + total_child_space;
+                    && target_offset < block_end
-                    if target_offset >= current_heap_offset && target_offset < block_end {
+                {
                     return self.read_from_indirect_block(
                         file_data,
                         child_addr as usize,
-                            child_nrows as u16,
+                        child_nrows,
                         current_heap_offset,
                         target_offset,
                         length,
@@ -442,11 +714,7 @@ impl FractalHeapHeader {
                         depth_remaining - 1,
                     );
                 }
-                    current_heap_offset += total_child_space;
+                current_heap_offset = block_end;
-                } else {
-                    let total_child_space = self.indirect_block_heap_size(child_nrows);
-                    current_heap_offset += total_child_space;
-                }
             }
         }
 
@@ -475,25 +743,34 @@ impl FractalHeapHeader {
         log2 + 2
     }
 
+    /// Rows an indirect block needs to span `size` bytes of heap space:
+    /// `log2(size) - log2(starting_block_size * table_width) + 1`, as
+    /// libhdf5's `H5HF__dtable_size_to_rows`.
+    fn rows_for_size(&self, size: u64) -> u16 {
+        let log2 = |v: u64| 63u32.saturating_sub(v.max(1).leading_zeros());
+        let first_row_bits = log2(self.starting_block_size) + log2(u64::from(self.table_width));
+        (log2(size).saturating_sub(first_row_bits) + 1) as u16
+    }
+
     /// Get block size for a given row in the doubling table.
     fn block_size_for_row(&self, row: usize) -> u64 {
         let sbs = self.starting_block_size;
         if row <= 1 {
             sbs
         } else {
-            sbs * (1u64 << (row - 1))
+            sbs.saturating_mul(1u64.checked_shl((row - 1) as u32).unwrap_or(u64::MAX))
+        }
     }
 }
 
-    /// Total heap space covered by an indirect block with the given number of rows.
+/// A managed direct block's location, extent and (for a filtered heap) its
-    fn indirect_block_heap_size(&self, nrows: usize) -> u64 {
+/// stored size and filter mask.
-        let tw = self.table_width as u64;
+struct DirectBlock {
-        let mut total = 0u64;
+    addr: usize,
-        for row in 0..nrows {
+    size: u64,
-            total += self.block_size_for_row(row) * tw;
+    heap_offset: u64,
-        }
+    filtered_size: u64,
-        total
+    filter_mask: u32,
-    }
 }
 
 #[cfg(test)]
@@ -641,7 +918,7 @@ mod tests {
         let hdr = FractalHeapHeader::parse(&file_data, 0, 8, 8).unwrap();
 
         // Build a managed heap ID:
-        // byte 0: type=0 (bits 6-7 = 00), version=0 (bits 4-5), reserved (bits 0-3)
+        // byte 0: version=0 (bits 6-7), type=0 (bits 4-5), reserved (bits 0-3)
         // bytes 1-6: offset (max_heap_size=16 bits) then length (remaining bits)
         // For offset=0, length=13:
         // payload = offset | (length << 16) = 0 | (13 << 16) = 0x000D0000
@@ -705,9 +982,46 @@ mod tests {
     fn invalid_heap_id_type() {
         let (file_data, _) = build_simple_heap(8, 8);
         let hdr = FractalHeapHeader::parse(&file_data, 0, 8, 8).unwrap();
-        // Type = 1 (tiny) in bits 6-7
+        // Type = 1 (huge) in bits 4-5 is not a managed ID
-        let id = vec![0x40u8, 0, 0, 0, 0, 0, 0]; // bit 6 set = type 1
+        let id = vec![0x10u8, 0, 0, 0, 0, 0, 0];
         let err = hdr.decode_managed_id(&id).unwrap_err();
         assert_eq!(err, FormatError::InvalidHeapIdType(1));
     }
+
+    #[test]
+    fn tiny_object_is_read_from_the_id() {
+        let (file_data, _) = build_simple_heap(8, 8);
+        let hdr = FractalHeapHeader::parse(&file_data, 0, 8, 8).unwrap();
+        // Type 2 (0x20), length - 1 in the low 4 bits, data after.
+        let id = [0x20 | 2, b'a', b'b', b'c', 0, 0, 0];
+        assert_eq!(hdr.read_managed_object(&file_data, &id, 8).unwrap(), b"abc");
+        // A length running past the ID is an error, not a short read.
+        let id = [0x20 | 9, b'a', b'b', b'c', 0, 0, 0];
+        assert!(hdr.read_managed_object(&file_data, &id, 8).is_err());
+    }
+
+    #[test]
+    fn huge_object_with_a_direct_id() {
+        // With IDs long enough for an address and a length, libhdf5 stores
+        // huge objects' location in the ID instead of the huge-object B-tree.
+        let (mut file_data, _) = build_simple_heap(8, 8);
+        let mut hdr = FractalHeapHeader::parse(&file_data, 0, 8, 8).unwrap();
+        hdr.heap_id_length = 17;
+        file_data[900..905].copy_from_slice(b"huge!");
+        let mut id = vec![0x10u8];
+        id.extend_from_slice(&900u64.to_le_bytes());
+        id.extend_from_slice(&5u64.to_le_bytes());
+        assert_eq!(
+            hdr.read_managed_object(&file_data, &id, 8).unwrap(),
+            b"huge!"
+        );
+    }
+
+    #[test]
+    fn unknown_heap_id_version_is_refused() {
+        let (file_data, _) = build_simple_heap(8, 8);
+        let hdr = FractalHeapHeader::parse(&file_data, 0, 8, 8).unwrap();
+        let id = [0x40u8, 0, 0, 0, 0, 0, 0];
+        assert!(hdr.read_managed_object(&file_data, &id, 8).is_err());
+    }
 }

crates/clawhdf5-format/src/group_v1.rs

@@ -45,9 +45,16 @@ pub fn resolve_v1_group_entries(
     )?;
 
     let mut entries = Vec::new();
+    let mut heap_checked = false;
     for snod_addr in snod_addrs {
         let snod = SymbolTableNode::parse(file_data, snod_addr as usize, offset_size)?;
         for entry in &snod.entries {
+            // Like libhdf5, look at the heap's free list only once a name is
+            // needed: an empty group with a damaged heap still lists.
+            if !heap_checked {
+                heap.validate_free_list(file_data, length_size)?;
+                heap_checked = true;
+            }
             let name = heap.read_string(file_data, entry.link_name_offset)?;
             entries.push(GroupEntry {
                 name,
@@ -73,6 +80,53 @@ pub fn find_v1_soft_link(
     offset_size: u8,
     length_size: u8,
 ) -> Result<Option<String>, FormatError> {
+    let mut found = None;
+    for_each_v1_soft_link(
+        file_data,
+        sym_table_msg,
+        offset_size,
+        length_size,
+        |link_name| link_name == name,
+        |_, target| {
+            found = Some(target);
+            false
+        },
+    )?;
+    Ok(found)
+}
+
+/// Every soft link in a v1 group, as `(name, target path)`.
+pub fn v1_soft_links(
+    file_data: &[u8],
+    sym_table_msg: &SymbolTableMessage,
+    offset_size: u8,
+    length_size: u8,
+) -> Result<Vec<(String, String)>, FormatError> {
+    let mut links = Vec::new();
+    for_each_v1_soft_link(
+        file_data,
+        sym_table_msg,
+        offset_size,
+        length_size,
+        |_| true,
+        |name, target| {
+            links.push((String::from(name), target));
+            true
+        },
+    )?;
+    Ok(links)
+}
+
+/// Visit the soft links of a v1 group whose name passes `wanted`, with their
+/// target paths, until `visit` returns false.
+fn for_each_v1_soft_link(
+    file_data: &[u8],
+    sym_table_msg: &SymbolTableMessage,
+    offset_size: u8,
+    length_size: u8,
+    wanted: impl Fn(&str) -> bool,
+    mut visit: impl FnMut(&str, String) -> bool,
+) -> Result<(), FormatError> {
     let heap = LocalHeap::parse(
         file_data,
         sym_table_msg.local_heap_address as usize,
@@ -85,13 +139,19 @@ pub fn find_v1_soft_link(
         offset_size,
         length_size,
     )?;
+    let mut heap_checked = false;
     for snod_addr in snod_addrs {
         let snod = SymbolTableNode::parse(file_data, snod_addr as usize, offset_size)?;
         for entry in &snod.entries {
             if entry.cache_type != CACHE_TYPE_SOFT_LINK {
                 continue;
             }
-            if heap.read_string(file_data, entry.link_name_offset)? != name {
+            if !heap_checked {
+                heap.validate_free_list(file_data, length_size)?;
+                heap_checked = true;
+            }
+            let name = heap.read_string(file_data, entry.link_name_offset)?;
+            if !wanted(&name) {
                 continue;
             }
             let value_offset = u32::from_le_bytes([
@@ -100,12 +160,19 @@ pub fn find_v1_soft_link(
                 entry.scratch_pad[2],
                 entry.scratch_pad[3],
             ]);
-            return heap
+            let target = heap.read_string(file_data, u64::from(value_offset))?;
-                .read_string(file_data, u64::from(value_offset))
+            if !visit(&name, target) {
-                .map(Some);
+                return Ok(());
             }
         }
-    Ok(None)
+    }
+    Ok(())
+}
+
+/// Whether a v1 symbol-table entry is a soft link (no object header of its
+/// own; its target path is in the local heap).
+pub fn is_v1_soft_link(entry: &GroupEntry) -> bool {
+    entry.cache_type == CACHE_TYPE_SOFT_LINK
 }
 
 /// Extract the SymbolTableMessage from an object header's messages.

crates/clawhdf5-format/src/group_v2.rs

@@ -38,6 +38,24 @@ pub fn resolve_v2_group_entries(
     }
 }
 
+/// First user-defined link type (HDF5 reserves 2-63; 64 is external).
+const FIRST_USER_DEFINED_LINK_TYPE: u8 = 65;
+
+/// Parse a Link message, or `None` for a user-defined link (type 65-255).
+///
+/// A user-defined link's target is only meaningful to the application that
+/// registered its class, so, like libhdf5 without that class, we cannot
+/// follow it. Leaving it out lets the rest of the group be listed and
+/// resolved instead of one such link failing the whole group; reserved
+/// types (2-63) are still an error.
+fn parse_link(data: &[u8], offset_size: u8) -> Result<Option<LinkMessage>, FormatError> {
+    match LinkMessage::parse(data, offset_size) {
+        Ok(link) => Ok(Some(link)),
+        Err(FormatError::InvalidLinkType(t)) if t >= FIRST_USER_DEFINED_LINK_TYPE => Ok(None),
+        Err(e) => Err(e),
+    }
+}
+
 /// Extract link entries from Link messages directly in the object header (compact storage).
 fn resolve_compact_entries(
     object_header: &ObjectHeader,
@@ -46,7 +64,9 @@ fn resolve_compact_entries(
     let mut entries = Vec::new();
     for msg in &object_header.messages {
         if msg.msg_type == MessageType::Link {
-            let link = LinkMessage::parse(&msg.data, offset_size)?;
+            let Some(link) = parse_link(&msg.data, offset_size)? else {
+                continue;
+            };
             if let LinkTarget::Hard {
                 object_header_address,
             } = link.link_target
@@ -98,7 +118,9 @@ fn for_each_dense_link(
 
         // Read managed object from fractal heap
         let link_data = fh.read_managed_object(file_data, id_bytes, offset_size)?;
-        visit(LinkMessage::parse(&link_data, offset_size)?);
+        if let Some(link) = parse_link(&link_data, offset_size)? {
+            visit(link);
+        }
     }
     Ok(())
 }
@@ -178,7 +200,9 @@ fn find_symbolic_link(
     } else {
         for msg in &object_header.messages {
             if msg.msg_type == MessageType::Link {
-                let link = LinkMessage::parse(&msg.data, offset_size)?;
+                let Some(link) = parse_link(&msg.data, offset_size)? else {
+                    continue;
+                };
                 if link.name == name && is_symbolic(&link.link_target) {
                     found = Some(link.link_target);
                 }
@@ -231,32 +255,135 @@ pub fn resolve_path_any(
     superblock: &Superblock,
     path: &str,
 ) -> Result<u64, FormatError> {
-    resolve_path_following_links(file_data, superblock, path, 0)
+    resolve_path_following_links(
+        file_data,
+        superblock,
+        superblock.root_group_address,
+        path,
+        0,
+    )
+}
+
+/// Resolve `path` relative to the group at `group_address` (an absolute path
+/// starts at the root group instead), following soft links. This is how a
+/// relative soft link's target is resolved: from the group holding the link.
+pub fn resolve_path_from(
+    file_data: &[u8],
+    superblock: &Superblock,
+    group_address: u64,
+    path: &str,
+) -> Result<u64, FormatError> {
+    let start = if path.starts_with('/') {
+        superblock.root_group_address
+    } else {
+        group_address
+    };
+    resolve_path_following_links(file_data, superblock, start, path, 0)
+}
+
+/// The children of the group at `group_address` that can be opened, as h5py
+/// lists them: hard links, and soft links resolved to the object they point
+/// at (under the soft link's own name). Links that cannot be followed are
+/// left out rather than failing the listing — a dangling or cyclic soft link
+/// (h5py lists its name but cannot open it), an external link (another
+/// file), and a user-defined link. An object header that is not a group has
+/// no children.
+///
+/// Any other error, such as a corrupt structure met while resolving a soft
+/// link, is returned.
+pub fn resolve_group_children(
+    file_data: &[u8],
+    superblock: &Superblock,
+    group_address: u64,
+) -> Result<Vec<GroupEntry>, FormatError> {
+    let os = superblock.offset_size;
+    let ls = superblock.length_size;
+    let header = ObjectHeader::parse(file_data, group_address as usize, os, ls)?;
+
+    let mut entries = Vec::new();
+    let mut soft = Vec::new();
+    if is_v1_group(&header) {
+        let sym_msg = header
+            .messages
+            .iter()
+            .find(|m| m.msg_type == MessageType::SymbolTable)
+            .ok_or_else(|| FormatError::PathNotFound(String::from("no symbol table message")))?;
+        let stm = SymbolTableMessage::parse(&sym_msg.data, os)?;
+        let all = group_v1::resolve_v1_group_entries(file_data, &stm, os, ls)?;
+        if all.iter().any(group_v1::is_v1_soft_link) {
+            soft = group_v1::v1_soft_links(file_data, &stm, os, ls)?;
+        }
+        entries.extend(all.into_iter().filter(|e| !group_v1::is_v1_soft_link(e)));
+    } else if is_v2_group(&header) {
+        let mut visit = |link: LinkMessage| match link.link_target {
+            LinkTarget::Hard {
+                object_header_address,
+            } => entries.push(GroupEntry {
+                name: link.name,
+                object_header_address,
+                cache_type: 0,
+            }),
+            LinkTarget::Soft { target_path } => soft.push((link.name, target_path)),
+            LinkTarget::External { .. } => {}
+        };
+        let link_info = find_link_info(&header, os)?;
+        if let Some(fh_addr) = link_info.fractal_heap_address {
+            for_each_dense_link(file_data, &link_info, fh_addr, os, ls, visit)?;
+        } else {
+            for msg in &header.messages {
+                if msg.msg_type == MessageType::Link
+                    && let Some(link) = parse_link(&msg.data, os)?
+                {
+                    visit(link);
+                }
+            }
+        }
+    }
+
+    for (name, target) in soft {
+        match resolve_path_from(file_data, superblock, group_address, &target) {
+            Ok(object_header_address) => entries.push(GroupEntry {
+                name,
+                object_header_address,
+                cache_type: 0,
+            }),
+            // Dangling, cyclic, or ending in another file: not openable here.
+            Err(
+                FormatError::PathNotFound(_)
+                | FormatError::NestingDepthExceeded
+                | FormatError::ExternalLinkUnsupported { .. },
+            ) => {}
+            Err(e) => return Err(e),
+        }
+    }
+    Ok(entries)
 }
 
 /// Soft links followed while resolving one path. Guards against link cycles
 /// (`a -> b -> a`), which are legal to create.
 const MAX_SOFT_LINK_DEPTH: u8 = 16;
 
+/// Walk `path` from the group at `start`, following soft links.
 fn resolve_path_following_links(
     file_data: &[u8],
     superblock: &Superblock,
+    start: u64,
     path: &str,
     depth: u8,
 ) -> Result<u64, FormatError> {
-    let components: Vec<&str> = path.split('/').filter(|s| !s.is_empty()).collect();
+    let components: Vec<&str> = path
+        .split('/')
+        .filter(|s| !s.is_empty() && *s != ".")
+        .collect();
     if components.is_empty() {
-        return Ok(superblock.root_group_address);
+        return Ok(start);
     }
 
     let os = superblock.offset_size;
     let ls = superblock.length_size;
 
-    let root_header =
+    let mut current_addr = start;
-        ObjectHeader::parse(file_data, superblock.root_group_address as usize, os, ls)?;
+    let mut current_header = ObjectHeader::parse(file_data, start as usize, os, ls)?;
-
-    let mut current_addr = superblock.root_group_address;
-    let mut current_header = root_header;
 
     for (i, component) in components.iter().enumerate() {
         let entries = resolve_group_entries(file_data, &current_header, os, ls)?;
@@ -280,20 +407,17 @@ fn resolve_path_following_links(
                         }
                         // A relative target is relative to the group holding
                         // the link; then the rest of the original path.
-                        let mut full = String::new();
+                        let from = if target_path.starts_with('/') {
-                        if !target_path.starts_with('/') {
+                            superblock.root_group_address
-                            for parent in &components[..i] {
+                        } else {
-                                full.push('/');
+                            current_addr
-                                full.push_str(parent);
+                        };
-                            }
+                        let mut full = target_path;
-                        }
-                        full.push('/');
-                        full.push_str(&target_path);
                         for rest in &components[i + 1..] {
                             full.push('/');
                             full.push_str(rest);
                         }
-                        resolve_path_following_links(file_data, superblock, &full, depth + 1)
+                        resolve_path_following_links(file_data, superblock, from, &full, depth + 1)
                     }
                     Some(LinkTarget::External {
                         filename,

crates/clawhdf5-format/src/lib.rs

@@ -26,12 +26,13 @@
 //! use clawhdf5_format::{signature, superblock, object_header, group_v2,
 //!     datatype, dataspace, data_layout, data_read, message_type::MessageType};
 //!
-//! let file_data = std::fs::read("output.h5").unwrap();
+//! let bytes = std::fs::read("output.h5").unwrap();
-//! let sig = signature::find_signature(&file_data).unwrap();
+//! // Addresses are relative to the superblock: skip any user block.
-//! let sb  = superblock::Superblock::parse(&file_data, sig).unwrap();
+//! let (_user_block, file_data) = signature::split_user_block(&bytes).unwrap();
-//! let addr = group_v2::resolve_path_any(&file_data, &sb, "data").unwrap();
+//! let sb  = superblock::Superblock::parse(file_data, 0).unwrap();
+//! let addr = group_v2::resolve_path_any(file_data, &sb, "data").unwrap();
 //! let hdr = object_header::ObjectHeader::parse(
-//!     &file_data, addr as usize, sb.offset_size, sb.length_size).unwrap();
+//!     file_data, addr as usize, sb.offset_size, sb.length_size).unwrap();
 //! ```
 //!
 //! # Features
@@ -42,6 +43,14 @@
 //! | `checksum` | yes | Jenkins lookup3 checksum validation |
 //! | `deflate` | yes | Deflate (gzip) compression via `flate2` |
 //! | `provenance` | yes | SHINES provenance — SHA-256 hashing & verification |
+//! | `lzf` | yes | LZF filter (32000), h5py's `compression="lzf"` |
+//! | `bitshuffle` | no | Bitshuffle filter (32008), none/LZ4/Zstandard |
+//! | `bzip2` | no | bzip2 filter (307) |
+//! | `blosc` | no | Blosc 1 filter (32001) |
+//! | `plugin-filters` | no | The four above |
+//!
+//! Filters are looked up by ID in [`filter_registry`], which also takes
+//! codecs registered at run time for other IDs.
 
 #![cfg_attr(not(feature = "std"), no_std)]
 
@@ -70,7 +79,16 @@ pub mod extensible_array;
 pub mod file_writer;
 pub mod fill_value;
 pub mod filter_pipeline;
+pub mod filter_registry;
 pub mod filters;
+#[cfg(any(feature = "bitshuffle", feature = "blosc"))]
+mod filters_bitshuffle;
+#[cfg(feature = "blosc")]
+pub mod filters_blosc;
+#[cfg(feature = "bzip2")]
+mod filters_bzip2;
+#[cfg(feature = "lzf")]
+pub mod filters_lzf;
 mod filters_szip;
 pub mod fixed_array;
 pub mod float16;
@@ -99,7 +117,18 @@ pub mod shared_message;
 pub mod signature;
 pub mod superblock;
 pub mod symbol_table;
+#[cfg(all(
+    test,
+    any(
+        feature = "lzf",
+        feature = "bitshuffle",
+        feature = "bzip2",
+        feature = "blosc"
+    )
+))]
+mod test_fuzz;
 pub mod type_builders;
+pub mod vds;
 pub mod vl_data;
 
 #[cfg(feature = "provenance")]

crates/clawhdf5-format/src/local_heap.rs

@@ -87,6 +87,57 @@ impl LocalHeap {
         })
     }
 
+    /// Walk the free list the way libhdf5 does when it loads a heap's data
+    /// (`H5HL__fl_deserialize`), rejecting a heap whose free list points
+    /// outside the data segment. libhdf5 refuses such a heap ("bad heap free
+    /// list"), and names read from it would be garbage.
+    ///
+    /// libhdf5 only loads a heap when it needs a name from it (an empty
+    /// group's broken heap goes unnoticed), so call this before the first
+    /// [`Self::read_string`], not on parse.
+    ///
+    /// The end of the list is `H5HL_FREE_NULL` (1); an all-ones value (the
+    /// undefined address) is accepted as "no free list" too.
+    pub fn validate_free_list(&self, file_data: &[u8], length_size: u8) -> Result<(), FormatError> {
+        const FREE_NULL: u64 = 1;
+        let ls = length_size as usize;
+        let undefined = if ls >= 8 {
+            u64::MAX
+        } else {
+            (1u64 << (8 * ls)) - 1
+        };
+        let size = self.data_segment_size;
+        let seg = self.data_segment_address;
+        let mut next = self.free_list_head_offset;
+        // Each free block holds two lengths, so a list longer than this
+        // revisits a block: a cycle.
+        let max_blocks = size / (2 * ls as u64) + 1;
+        let mut walked = 0u64;
+        while next != FREE_NULL && next != undefined {
+            if next >= size || walked >= max_blocks {
+                return Err(FormatError::InvalidLocalHeapFreeList);
+            }
+            walked += 1;
+            let at = seg
+                .checked_add(next)
+                .and_then(|a| usize::try_from(a).ok())
+                .ok_or(FormatError::InvalidLocalHeapFreeList)?;
+            let block_offset = next;
+            next = read_offset(file_data, at, length_size)?;
+            if next == 0 {
+                return Err(FormatError::InvalidLocalHeapFreeList);
+            }
+            let block_size = read_offset(file_data, at + ls, length_size)?;
+            if block_offset
+                .checked_add(block_size)
+                .is_none_or(|end| end > size)
+            {
+                return Err(FormatError::InvalidLocalHeapFreeList);
+            }
+        }
+        Ok(())
+    }
+
     /// Read a null-terminated string from the heap's data segment at the given byte offset.
     pub fn read_string(&self, file_data: &[u8], string_offset: u64) -> Result<String, FormatError> {
         let seg_addr = self.data_segment_address as usize;
@@ -162,8 +213,8 @@ mod tests {
         // data_segment_size
         write_val(&mut file, pos, data_seg_size as u64, length_size);
         pos += length_size as usize;
-        // free_list_head_offset
+        // free_list_head_offset: H5HL_FREE_NULL (no free space)
-        write_val(&mut file, pos, 0xFFFFFFFF, length_size);
+        write_val(&mut file, pos, 1, length_size);
         pos += length_size as usize;
         // data_segment_address
         write_val(&mut file, pos, data_seg_offset as u64, offset_size);
@@ -243,6 +294,50 @@ mod tests {
         assert_eq!(s, "test");
     }
 
+    /// Heap with data segment `[a, b, c, 0-padding]` whose free list starts
+    /// at `head` and has one block `(next, size)` at offset 8.
+    fn heap_with_free_block(head: u64, next: u64, size: u64) -> Vec<u8> {
+        let mut file = build_heap_file(0, 100, &["abcdefg"], 8, 8);
+        file.resize(200, 0);
+        write_val(&mut file, 8, 32, 8); // data segment size
+        write_val(&mut file, 16, head, 8);
+        write_val(&mut file, 108, next, 8);
+        write_val(&mut file, 116, size, 8);
+        file
+    }
+
+    #[test]
+    fn free_list_inside_the_segment_is_accepted() {
+        let file = heap_with_free_block(8, 1, 24);
+        let heap = LocalHeap::parse(&file, 0, 8, 8).unwrap();
+        heap.validate_free_list(&file, 8).unwrap();
+        assert_eq!(heap.read_string(&file, 0).unwrap(), "abcdefg");
+        // An all-ones head is "no free list" too.
+        let file = heap_with_free_block(u64::MAX, 0, 0);
+        let heap = LocalHeap::parse(&file, 0, 8, 8).unwrap();
+        assert!(heap.validate_free_list(&file, 8).is_ok());
+    }
+
+    #[test]
+    fn bad_free_list_is_rejected_like_libhdf5() {
+        for (head, next, size, why) in [
+            (40, 1, 8, "head past the segment"),
+            (8, 1, 25, "block runs past the segment"),
+            (8, 0, 8, "next offset of zero"),
+            (8, 8, 8, "cycle"),
+            (8, 999, 8, "next past the segment"),
+        ] {
+            let file = heap_with_free_block(head, next, size);
+            // The header itself parses; the free list is checked on use.
+            let heap = LocalHeap::parse(&file, 0, 8, 8).unwrap();
+            assert_eq!(
+                heap.validate_free_list(&file, 8).unwrap_err(),
+                FormatError::InvalidLocalHeapFreeList,
+                "{why}"
+            );
+        }
+    }
+
     #[test]
     fn invalid_version() {
         let mut file = build_heap_file(0, 100, &["x"], 8, 8);

crates/clawhdf5-format/src/object_header.rs

@@ -108,10 +108,20 @@ impl ObjectHeader {
             return Err(FormatError::InvalidObjectHeaderVersion(version));
         }
 
-        let num_messages = LittleEndian::read_u16(&data[offset + 2..offset + 4]);
+        let num_messages = LittleEndian::read_u16(&data[offset + 2..offset + 4]) as usize;
         let reference_count = LittleEndian::read_u32(&data[offset + 4..offset + 8]);
         let header_data_size = LittleEndian::read_u32(&data[offset + 8..offset + 12]) as usize;
 
+        // libhdf5 (H5O__prefix_deserialize): a header with messages needs room
+        // for at least one message header, and one without has an empty chunk.
+        if (num_messages > 0 && header_data_size < V1_MSG_HEADER_SIZE)
+            || (num_messages == 0 && header_data_size > 0)
+        {
+            return Err(FormatError::InvalidObjectHeader(
+                "bad object header chunk size",
+            ));
+        }
+
         // Pad to 8-byte alignment: header prefix is 12 bytes, pad to 16
         let padding = 4; // pad 12-byte prefix to 16-byte alignment
         let msg_start = offset
@@ -124,64 +134,23 @@ impl ObjectHeader {
         ensure_len(data, msg_start, header_data_size)?;
 
         let mut messages = Vec::new();
-        let mut pos = msg_start;
+        let chunk0_count = Self::parse_v1_chunk(
-        let msg_end =
-            msg_start
-                .checked_add(header_data_size)
-                .ok_or(FormatError::UnexpectedEof {
-                    expected: usize::MAX,
-                    available: data.len(),
-                })?;
-
-        for _ in 0..num_messages {
-            if pos + 8 > msg_end {
-                break;
-            }
-            let msg_type_raw = LittleEndian::read_u16(&data[pos..pos + 2]);
-            let msg_data_size = LittleEndian::read_u16(&data[pos + 2..pos + 4]) as usize;
-            let msg_flags = data[pos + 4];
-            // reserved(3) at pos+5..pos+8
-            pos += 8;
-
-            ensure_len(data, pos, msg_data_size)?;
-            let msg_type = MessageType::from_u16(msg_type_raw);
-
-            check_unknown_message(msg_type, msg_flags)?;
-
-            if msg_type != MessageType::Nil {
-                messages.push(HeaderMessage {
-                    msg_type,
-                    size: msg_data_size,
-                    flags: msg_flags,
-                    creation_order: None,
-                    data: data[pos..pos + msg_data_size].to_vec(),
-                });
-            }
-
-            pos += msg_data_size;
-
-            // Follow continuations
-            if msg_type == MessageType::ObjectHeaderContinuation {
-                let cont_msg_data = &messages
-                    .last()
-                    .ok_or(FormatError::InvalidObjectHeaderSignature)?
-                    .data;
-                if cont_msg_data.len() >= (offset_size as usize + length_size as usize) {
-                    let cont_offset = read_offset(cont_msg_data, 0, offset_size)? as usize;
-                    let cont_length =
-                        read_offset(cont_msg_data, offset_size as usize, length_size)? as usize;
-                    // Parse continuation block (v1: just raw messages, no signature)
-                    let cont_msgs = Self::parse_v1_continuation(
             data,
-                        cont_offset,
+            msg_start,
-                        cont_length,
+            header_data_size,
             offset_size,
             length_size,
-                        32, // max continuation depth
+            MAX_V1_CONTINUATION_DEPTH,
+            &mut messages,
         )?;
-                    messages.extend(cont_msgs);
+        // libhdf5 reads every message in the first chunk and refuses a header
-                }
+        // whose prefix claims fewer than that (continuation chunks are read
-            }
+        // later and not held to the count). Stopping after the claimed number
+        // silently dropped the rest.
+        if chunk0_count > num_messages {
+            return Err(FormatError::InvalidObjectHeader(
+                "bad object header message count",
+            ));
         }
 
         Ok(ObjectHeader {
@@ -196,72 +165,87 @@ impl ObjectHeader {
         })
     }
 
-    fn parse_v1_continuation(
+    /// Parse the messages of one version-1 chunk (`length` bytes at
+    /// `offset`, no signature), following continuation messages as they are
+    /// met. Returns how many messages (NIL ones included) this chunk itself
+    /// holds.
+    ///
+    /// A version-1 chunk is filled with messages whose sizes are multiples of
+    /// 8; libhdf5 refuses a message that is not aligned, that runs past the
+    /// end of the chunk, or leftover bytes too few for a message header (a
+    /// "gap", which only version 2 allows).
+    #[allow(clippy::too_many_arguments)]
+    fn parse_v1_chunk(
         data: &[u8],
         offset: usize,
         length: usize,
         offset_size: u8,
         length_size: u8,
         depth_remaining: u16,
-    ) -> Result<Vec<HeaderMessage>, FormatError> {
+        messages: &mut Vec<HeaderMessage>,
+    ) -> Result<usize, FormatError> {
         if depth_remaining == 0 {
             return Err(FormatError::NestingDepthExceeded);
         }
         ensure_len(data, offset, length)?;
-        let mut messages = Vec::new();
+        let end = offset + length;
         let mut pos = offset;
-        let end = offset.saturating_add(length);
+        let mut count = 0usize;
 
-        while pos + 8 <= end {
+        while pos < end {
+            if end - pos < V1_MSG_HEADER_SIZE {
+                return Err(FormatError::InvalidObjectHeader(
+                    "gap found in early version of file format",
+                ));
+            }
             let msg_type_raw = LittleEndian::read_u16(&data[pos..pos + 2]);
             let msg_data_size = LittleEndian::read_u16(&data[pos + 2..pos + 4]) as usize;
             let msg_flags = data[pos + 4];
-            pos += 8;
+            // reserved(3) at pos+5..pos+8
+            pos += V1_MSG_HEADER_SIZE;
 
-            if pos + msg_data_size > end {
+            if !msg_data_size.is_multiple_of(8) {
-                break;
+                return Err(FormatError::InvalidObjectHeader("message not aligned"));
             }
+            if msg_data_size > end - pos {
+                return Err(FormatError::InvalidObjectHeader(
+                    "message size exceeds buffer end",
+                ));
+            }
+            let body = &data[pos..pos + msg_data_size];
+            check_message(1, msg_type_raw, msg_flags, body, offset_size, length_size)?;
+            count += 1;
 
             let msg_type = MessageType::from_u16(msg_type_raw);
-
-            check_unknown_message(msg_type, msg_flags)?;
-
             if msg_type != MessageType::Nil {
                 messages.push(HeaderMessage {
                     msg_type,
                     size: msg_data_size,
                     flags: msg_flags,
                     creation_order: None,
-                    data: data[pos..pos + msg_data_size].to_vec(),
+                    data: body.to_vec(),
                 });
             }
-
             pos += msg_data_size;
 
-            // Recursive continuations
+            // Follow continuations (v1 continuation chunks are just raw
+            // messages, no signature); check_message has checked the body.
             if msg_type == MessageType::ObjectHeaderContinuation {
-                let cont_msg_data = &messages
+                let cont_offset = read_offset(body, 0, offset_size)? as usize;
-                    .last()
+                let cont_length = read_offset(body, offset_size as usize, length_size)? as usize;
-                    .ok_or(FormatError::InvalidObjectHeaderSignature)?
+                Self::parse_v1_chunk(
-                    .data;
-                if cont_msg_data.len() >= (offset_size as usize + length_size as usize) {
-                    let cont_offset = read_offset(cont_msg_data, 0, offset_size)? as usize;
-                    let cont_length =
-                        read_offset(cont_msg_data, offset_size as usize, length_size)? as usize;
-                    let cont_msgs = Self::parse_v1_continuation(
                     data,
                     cont_offset,
                     cont_length,
                     offset_size,
                     length_size,
                     depth_remaining - 1,
+                    messages,
                 )?;
-                    messages.extend(cont_msgs);
-                }
             }
         }
 
-        Ok(messages)
+        Ok(count)
     }
 
     fn parse_v2(
@@ -278,6 +262,11 @@ impl ObjectHeader {
             return Err(FormatError::InvalidObjectHeaderVersion(version));
         }
         let flags = data[offset + 5];
+        if flags & !V2_HDR_ALL_FLAGS != 0 {
+            return Err(FormatError::InvalidObjectHeader(
+                "unknown object header status flag(s)",
+            ));
+        }
 
         let mut pos = offset + 6;
 
@@ -297,7 +286,14 @@ impl ObjectHeader {
         // Optional attribute storage thresholds (flags bit 4)
         if flags & 0x10 != 0 {
             ensure_len(data, pos, 4)?;
-            // max_compact_attrs(2) + min_dense_attrs(2) — read but don't store for now
+            // max_compact_attrs(2) + min_dense_attrs(2) — checked, not stored
+            let max_compact = LittleEndian::read_u16(&data[pos..pos + 2]);
+            let min_dense = LittleEndian::read_u16(&data[pos + 2..pos + 4]);
+            if max_compact < min_dense {
+                return Err(FormatError::InvalidObjectHeader(
+                    "bad object header attribute phase change values",
+                ));
+            }
             pos += 4;
         }
 
@@ -312,6 +308,14 @@ impl ObjectHeader {
         ensure_len(data, pos, chunk_size_width as usize)?;
         let chunk0_size = read_offset(data, pos, chunk_size_width)? as usize;
         pos += chunk_size_width as usize;
+        // Bit 2: attribute creation order tracked → messages include creation order field
+        let has_creation_order = flags & 0x04 != 0;
+        let msg_header_size = if has_creation_order { 6 } else { 4 };
+        if chunk0_size > 0 && chunk0_size < msg_header_size {
+            return Err(FormatError::InvalidObjectHeader(
+                "bad object header chunk size",
+            ));
+        }
 
         let chunk0_msg_start = pos;
         let chunk0_msg_end = pos
@@ -335,9 +339,6 @@ impl ObjectHeader {
             }
         }
 
-        // Bit 2: attribute creation order tracked → messages include creation order field
-        let has_creation_order = flags & 0x04 != 0;
-
         // Parse messages from chunk0
         let mut messages = Vec::new();
         let mut continuations = Vec::new();
@@ -396,8 +397,20 @@ impl ObjectHeader {
     ) -> Result<(), FormatError> {
         let msg_header_size = if has_creation_order { 6 } else { 4 };
         let mut pos = start;
+        let mut null_count = 0usize;
 
-        while pos + msg_header_size <= end {
+        while pos < end {
+            // Leftover bytes too few for a message header are a gap, which
+            // libhdf5 allows only in a chunk without NIL messages (a writer
+            // that leaves a gap had no NIL message to put the space in).
+            if end - pos < msg_header_size {
+                if null_count != 0 {
+                    return Err(FormatError::InvalidObjectHeader(
+                        "gap in chunk with no null messages",
+                    ));
+                }
+                break;
+            }
             let msg_type_raw = data[pos] as u16;
             let msg_data_size = LittleEndian::read_u16(&data[pos + 1..pos + 3]) as usize;
             let msg_flags = data[pos + 3];
@@ -408,32 +421,36 @@ impl ObjectHeader {
             };
             pos += msg_header_size;
 
-            if pos + msg_data_size > end {
+            // `end` is where the messages stop and the checksum starts.
-                // Could be padding at end of chunk
+            // libhdf5 bounds a message by the chunk including its checksum,
-                break;
+            // but a message that runs into the checksum still fails there:
+            // its loop stops at the checksum, and reading the checksum from
+            // past its start overruns the chunk ("ran off end of input
+            // buffer while decoding"). Both refuse it; only the text
+            // differs.
+            if msg_data_size > end - pos {
+                return Err(FormatError::InvalidObjectHeader(
+                    "message size exceeds buffer end",
+                ));
             }
+            let body = &data[pos..pos + msg_data_size];
+            check_message(2, msg_type_raw, msg_flags, body, offset_size, length_size)?;
 
             let msg_type = MessageType::from_u16(msg_type_raw);
-
-            check_unknown_message(msg_type, msg_flags)?;
-
-            let msg_data = data[pos..pos + msg_data_size].to_vec();
-
             if msg_type == MessageType::ObjectHeaderContinuation {
-                // Parse continuation offset/length from message data
+                // check_message has checked the body holds both fields.
-                if msg_data.len() >= (offset_size as usize + length_size as usize) {
+                let cont_off = read_offset(body, 0, offset_size)? as usize;
-                    let cont_off = read_offset(&msg_data, 0, offset_size)? as usize;
+                let cont_len = read_offset(body, offset_size as usize, length_size)? as usize;
-                    let cont_len =
-                        read_offset(&msg_data, offset_size as usize, length_size)? as usize;
                 continuations.push((cont_off, cont_len));
-                }
+            } else if msg_type == MessageType::Nil {
-            } else if msg_type != MessageType::Nil {
+                null_count += 1;
+            } else {
                 messages.push(HeaderMessage {
                     msg_type,
                     size: msg_data_size,
                     flags: msg_flags,
                     creation_order,
-                    data: msg_data,
+                    data: body.to_vec(),
                 });
             }
 
@@ -496,22 +513,149 @@ impl ObjectHeader {
     }
 }
 
-/// Header message flag bit 7: fail if the message is unknown, always.
+/// Size of a version-1 message header: type(2) + size(2) + flags(1) + reserved(3).
+const V1_MSG_HEADER_SIZE: usize = 8;
+
+/// How deep version-1 continuation chunks may chain (malformed-data guard).
+const MAX_V1_CONTINUATION_DEPTH: u16 = 32;
+
+/// Every defined version-2 object header status flag (libhdf5
+/// `H5O_HDR_ALL_FLAGS`): chunk-0 size width (bits 0-1), attribute creation
+/// order tracked/indexed, attribute phase-change values, times stored.
+const V2_HDR_ALL_FLAGS: u8 = 0x3F;
+
+// Header message flag bits (libhdf5 `H5O_MSG_FLAG_*`). Bit 0 (constant) needs
+// no check. Bit 3 (fail if unknown and the file is opened for writing) never
+// fails a read: the parser only ever reads, as libhdf5 ignores it for a
+// read-only open.
+const MSG_FLAG_SHARED: u8 = 0x02;
+const MSG_FLAG_DONTSHARE: u8 = 0x04;
+const MSG_FLAG_FAIL_IF_UNKNOWN_AND_OPEN_FOR_WRITE: u8 = 0x08;
+const MSG_FLAG_MARK_IF_UNKNOWN: u8 = 0x10;
+const MSG_FLAG_WAS_UNKNOWN: u8 = 0x20;
+const MSG_FLAG_SHAREABLE: u8 = 0x40;
+/// Fail if the message is unknown, whatever the access mode.
 const MSG_FLAG_FAIL_IF_UNKNOWN_ALWAYS: u8 = 0x80;
 
-/// Refuse an unknown message the file says no reader may skip.
+/// Message type ids libhdf5 has a class for (`H5O_msg_class_g`): 0x00-0x18
+/// except 0x09 (a test-only "bogus" message). Anything else is an unknown
+/// message.
+fn is_known_message(id: u16) -> bool {
+    id <= 0x18 && id != 0x09
+}
+
+/// Message classes that may be shared (`H5O_SHARE_IS_SHARABLE`): dataspace,
+/// datatype, the two fill-value messages, filter pipeline and attribute.
+fn is_shareable_message(id: u16) -> bool {
+    matches!(id, 0x01 | 0x03 | 0x04 | 0x05 | 0x0B | 0x0C)
+}
+
+/// Check one header message the way libhdf5 does while it loads an object
+/// header (`H5O__chunk_deserialize`), so an object libhdf5 refuses to open is
+/// refused here too instead of being read from a corrupt header:
 ///
-/// The parser only ever reads, so bit 3 (fail only when opened for writing)
+/// - contradictory flag combinations;
-/// is ignored, as libhdf5 ignores it for a read-only open; bit 7 fails
+/// - an unknown message the file says no reader may skip (bit 7). This had
-/// regardless of access mode. This had the two the wrong way round, failing
+///   bits 3 and 7 the wrong way round once, failing objects libhdf5 reads
-/// objects libhdf5 reads and reading ones it refuses (`tbogus.h5`).
+///   and reading ones it refuses (`tbogus.h5`);
-fn check_unknown_message(msg_type: MessageType, msg_flags: u8) -> Result<(), FormatError> {
+/// - a known message whose class cannot be shared, flagged shared or
-    match msg_type {
+///   shareable (`cve-2016-4332`);
-        MessageType::Unknown(id) if msg_flags & MSG_FLAG_FAIL_IF_UNKNOWN_ALWAYS != 0 => {
+/// - the messages libhdf5 decodes while loading the header, whose decode
-            Err(FormatError::UnsupportedMessage(id))
+///   errors fail the load: continuation, reference count (which a version-1
+///   header cannot hold), and both modification-time messages.
+fn check_message(
+    header_version: u8,
+    id: u16,
+    flags: u8,
+    body: &[u8],
+    offset_size: u8,
+    length_size: u8,
+) -> Result<(), FormatError> {
+    let bad_flags = FormatError::InvalidObjectHeader("bad flag combination for message");
+    if flags & MSG_FLAG_SHARED != 0 && flags & MSG_FLAG_DONTSHARE != 0 {
+        return Err(bad_flags);
     }
-        _ => Ok(()),
+    if flags & MSG_FLAG_WAS_UNKNOWN != 0
+        && (flags & MSG_FLAG_FAIL_IF_UNKNOWN_AND_OPEN_FOR_WRITE != 0
+            || flags & MSG_FLAG_MARK_IF_UNKNOWN == 0)
+    {
+        return Err(bad_flags);
     }
+
+    if !is_known_message(id) {
+        if flags & MSG_FLAG_FAIL_IF_UNKNOWN_ALWAYS != 0 {
+            return Err(FormatError::UnsupportedMessage(id));
+        }
+        return Ok(());
+    }
+    if flags & (MSG_FLAG_SHARED | MSG_FLAG_SHAREABLE) != 0 && !is_shareable_message(id) {
+        return Err(FormatError::InvalidObjectHeader(
+            "message of unshareable class flagged as shareable",
+        ));
+    }
+
+    let overrun = FormatError::InvalidObjectHeader("ran off end of input buffer while decoding");
+    match id {
+        // Continuation: address + length, and the chunk cannot be empty.
+        0x10 => {
+            if body.len() < offset_size as usize + length_size as usize {
+                return Err(overrun);
+            }
+            if read_offset(body, offset_size as usize, length_size)? == 0 {
+                return Err(FormatError::InvalidObjectHeader(
+                    "invalid continuation chunk size (0)",
+                ));
+            }
+        }
+        // Reference count: version-2 headers only; version 0 then a u32.
+        0x16 => {
+            if header_version == 1 {
+                return Err(FormatError::InvalidObjectHeader(
+                    "object header version does not support reference count message",
+                ));
+            }
+            match body.first() {
+                None => return Err(overrun),
+                Some(0) => {}
+                Some(_) => {
+                    return Err(FormatError::InvalidObjectHeader(
+                        "bad version number for reference count message",
+                    ));
+                }
+            }
+            if body.len() < 5 {
+                return Err(overrun);
+            }
+        }
+        // Old modification time: "YYYYMMDDhhmmss" and 2 reserved bytes.
+        0x0E => {
+            if body.len() < 16 {
+                return Err(overrun);
+            }
+            if !body[..14].iter().all(u8::is_ascii_digit) {
+                return Err(FormatError::InvalidObjectHeader(
+                    "badly formatted modification time message",
+                ));
+            }
+        }
+        // New modification time: version 1, 3 reserved bytes, u32 seconds.
+        0x12 => {
+            match body.first() {
+                None => return Err(overrun),
+                Some(1) => {}
+                Some(_) => {
+                    return Err(FormatError::InvalidObjectHeader(
+                        "bad version number for mtime message",
+                    ));
+                }
+            }
+            if body.len() < 8 {
+                return Err(overrun);
+            }
+        }
+        _ => {}
+    }
+    Ok(())
 }
 
 #[cfg(test)]
@@ -528,11 +672,14 @@ mod tests {
         // Calculate total header message data size
         let mut msg_bytes = Vec::new();
         for (mtype, mdata, mflags) in messages {
+            // v1 message sizes are multiples of 8 (the data is zero-padded).
+            let padded = mdata.len().div_ceil(8) * 8;
             msg_bytes.extend_from_slice(&mtype.to_le_bytes()); // type(2)
-            msg_bytes.extend_from_slice(&(mdata.len() as u16).to_le_bytes()); // size(2)
+            msg_bytes.extend_from_slice(&(padded as u16).to_le_bytes()); // size(2)
             msg_bytes.push(*mflags); // flags(1)
             msg_bytes.extend_from_slice(&[0u8; 3]); // reserved(3)
             msg_bytes.extend_from_slice(mdata); // data
+            msg_bytes.resize(msg_bytes.len() + padded - mdata.len(), 0);
         }
 
         let mut buf = Vec::new();
@@ -622,9 +769,10 @@ mod tests {
         let hdr = ObjectHeader::parse(&data, 0, 8, 8).unwrap();
         assert_eq!(hdr.messages.len(), 2);
         assert_eq!(hdr.messages[0].msg_type, MessageType::Dataspace);
-        assert_eq!(hdr.messages[0].data, vec![1, 2, 3, 4]);
+        // v1 message data is padded to a multiple of 8 bytes.
+        assert_eq!(hdr.messages[0].data, vec![1, 2, 3, 4, 0, 0, 0, 0]);
         assert_eq!(hdr.messages[1].msg_type, MessageType::DataLayout);
-        assert_eq!(hdr.messages[1].data, vec![5, 6]);
+        assert_eq!(hdr.messages[1].data[..2], [5, 6]);
     }
 
     #[test]
@@ -650,7 +798,7 @@ mod tests {
         // Bit 3 = fail if unknown *and the file is opened for writing*. This
         // parser only reads, so libhdf5 (read-only) opens such an object and
         // so must we. Bits 4/5 (mark if unknown / was unknown) never fail.
-        for flags in [0x08u8, 0x10, 0x20, 0x38] {
+        for flags in [0x08u8, 0x10, 0x30] {
             let messages = [(0x00FFu16, &[0xAA][..], flags)];
             let data = build_v1_header(&messages, 8, 8);
             let hdr = ObjectHeader::parse(&data, 0, 8, 8).unwrap();
@@ -658,6 +806,231 @@ mod tests {
         }
     }
 
+    #[test]
+    fn contradictory_message_flags_are_refused() {
+        // libhdf5: "bad flag combination for message" for shared + don't
+        // share, was-unknown without mark-if-unknown, and was-unknown with
+        // fail-if-unknown-on-write.
+        for flags in [0x06u8, 0x20, 0x38] {
+            let data = build_v1_header(&[(0x00FFu16, &[0xAA][..], flags)], 8, 8);
+            assert_eq!(
+                ObjectHeader::parse(&data, 0, 8, 8).unwrap_err(),
+                FormatError::InvalidObjectHeader("bad flag combination for message"),
+                "flags {flags:#x}"
+            );
+            let data = build_v2_header(0x00, &[(0xF0, &[1, 2], flags)], None);
+            assert_eq!(
+                ObjectHeader::parse(&data, 0, 8, 8).unwrap_err(),
+                FormatError::InvalidObjectHeader("bad flag combination for message"),
+                "flags {flags:#x}"
+            );
+        }
+    }
+
+    #[test]
+    fn unshareable_message_flagged_shareable_is_refused() {
+        // A layout (0x08) or modification time (0x12) message cannot be
+        // shared; bit 1 (shared) or bit 6 (shareable) on one is corruption
+        // (cve-2016-4332). A datatype (0x03) may be shareable.
+        let mtime = [1u8, 0, 0, 0, 0x10, 0x20, 0x30, 0x40];
+        for (id, flags) in [(0x08u16, 0x40u8), (0x08, 0x02), (0x12, 0x40)] {
+            let data = build_v1_header(&[(id, &mtime[..], flags)], 8, 8);
+            assert_eq!(
+                ObjectHeader::parse(&data, 0, 8, 8).unwrap_err(),
+                FormatError::InvalidObjectHeader(
+                    "message of unshareable class flagged as shareable"
+                ),
+                "id {id:#x} flags {flags:#x}"
+            );
+        }
+        let data = build_v1_header(&[(0x03, &[0u8; 8][..], 0x40)], 8, 8);
+        assert!(ObjectHeader::parse(&data, 0, 8, 8).is_ok());
+        // An unknown message is never checked for shareability.
+        let data = build_v1_header(&[(0x00FF, &[0u8; 8][..], 0x40)], 8, 8);
+        assert!(ObjectHeader::parse(&data, 0, 8, 8).is_ok());
+    }
+
+    #[test]
+    fn v1_message_must_be_aligned() {
+        // cve-2018-13873: a v1 message whose size is not a multiple of 8.
+        let mut data = build_v1_header(&[(0x01, &[0u8; 8][..], 0)], 8, 8);
+        data[16 + 2] = 7; // size field of the only message
+        assert_eq!(
+            ObjectHeader::parse(&data, 0, 8, 8).unwrap_err(),
+            FormatError::InvalidObjectHeader("message not aligned")
+        );
+    }
+
+    #[test]
+    fn message_overrunning_its_chunk_is_refused() {
+        // It used to end the chunk quietly, dropping this message and any
+        // after it.
+        let mut data = build_v1_header(&[(0x01, &[0u8; 8][..], 0)], 8, 8);
+        data[16 + 2] = 16;
+        data.resize(data.len() + 64, 0);
+        assert_eq!(
+            ObjectHeader::parse(&data, 0, 8, 8).unwrap_err(),
+            FormatError::InvalidObjectHeader("message size exceeds buffer end")
+        );
+        let mut data = build_v2_header(0x00, &[(0x01, &[1, 2], 0)], None);
+        data[7 + 1] = 9; // size of the only message (after OHDR, ver, flags, chunk size)
+        let chk = crate::checksum::jenkins_lookup3(&data[..data.len() - 4]);
+        let n = data.len();
+        data[n - 4..].copy_from_slice(&chk.to_le_bytes());
+        assert_eq!(
+            ObjectHeader::parse(&data, 0, 8, 8).unwrap_err(),
+            FormatError::InvalidObjectHeader("message size exceeds buffer end")
+        );
+    }
+
+    #[test]
+    fn v1_gap_after_last_message_is_refused() {
+        // Fewer than 8 bytes left over: a gap, which only version 2 allows.
+        let mut data = build_v1_header(&[(0x01, &[0u8; 8][..], 0)], 8, 8);
+        data[8] += 4; // header_data_size
+        data.extend_from_slice(&[0u8; 4]);
+        assert_eq!(
+            ObjectHeader::parse(&data, 0, 8, 8).unwrap_err(),
+            FormatError::InvalidObjectHeader("gap found in early version of file format")
+        );
+    }
+
+    #[test]
+    fn v1_chunk_holding_more_messages_than_the_prefix_says_is_refused() {
+        // cve-2024-32619: the prefix says 1 message, the chunk holds 2. The
+        // second used to be dropped silently.
+        let mut data = build_v1_header(&[(0x01, &[0u8; 8][..], 0), (0x03, &[0u8; 8][..], 0)], 8, 8);
+        data[2] = 1;
+        assert_eq!(
+            ObjectHeader::parse(&data, 0, 8, 8).unwrap_err(),
+            FormatError::InvalidObjectHeader("bad object header message count")
+        );
+        // Fewer in the chunk than the prefix says is fine (the rest may be in
+        // continuation chunks; libhdf5 only enforces that with strict checks).
+        data[2] = 3;
+        assert_eq!(
+            ObjectHeader::parse(&data, 0, 8, 8).unwrap().messages.len(),
+            2
+        );
+    }
+
+    #[test]
+    fn v1_prefix_chunk_size_must_fit_the_message_count() {
+        let mut data = build_v1_header(&[], 8, 8);
+        data[8] = 8; // no messages but a non-empty chunk
+        data.extend_from_slice(&[0u8; 8]);
+        assert_eq!(
+            ObjectHeader::parse(&data, 0, 8, 8).unwrap_err(),
+            FormatError::InvalidObjectHeader("bad object header chunk size")
+        );
+    }
+
+    #[test]
+    fn v1_header_cannot_hold_a_reference_count_message() {
+        // cve-2018-11204.
+        let data = build_v1_header(&[(0x16, &[0, 2, 0, 0, 0][..], 0)], 8, 8);
+        assert_eq!(
+            ObjectHeader::parse(&data, 0, 8, 8).unwrap_err(),
+            FormatError::InvalidObjectHeader(
+                "object header version does not support reference count message"
+            )
+        );
+        let data = build_v2_header(0x00, &[(0x16, &[0, 2, 0, 0, 0], 0)], None);
+        assert!(ObjectHeader::parse(&data, 0, 8, 8).is_ok());
+    }
+
+    #[test]
+    fn modification_time_messages_are_decoded_with_the_header() {
+        // cve-2024-33873 (version 0) and cve-2024-33874 (empty message).
+        for (body, why) in [
+            (
+                &[0u8, 0, 0, 0, 1, 2, 3, 4][..],
+                "bad version number for mtime message",
+            ),
+            (&[][..], "ran off end of input buffer while decoding"),
+        ] {
+            let data = build_v2_header(0x00, &[(0x12, body, 0)], None);
+            assert_eq!(
+                ObjectHeader::parse(&data, 0, 8, 8).unwrap_err(),
+                FormatError::InvalidObjectHeader(why)
+            );
+        }
+        let data = build_v2_header(0x00, &[(0x12, &[1, 0, 0, 0, 1, 2, 3, 4], 0)], None);
+        assert!(ObjectHeader::parse(&data, 0, 8, 8).is_ok());
+        // The old (0x0E) message is 14 ASCII digits and 2 reserved bytes.
+        let data = build_v1_header(&[(0x0E, &b"20110414214255\0\0"[..], 0)], 8, 8);
+        assert!(ObjectHeader::parse(&data, 0, 8, 8).is_ok());
+        let data = build_v1_header(&[(0x0E, &b"2011041421425x\0\0"[..], 0)], 8, 8);
+        assert_eq!(
+            ObjectHeader::parse(&data, 0, 8, 8).unwrap_err(),
+            FormatError::InvalidObjectHeader("badly formatted modification time message")
+        );
+    }
+
+    #[test]
+    fn continuation_message_must_hold_a_nonempty_chunk() {
+        let mut cont = [0u8; 16];
+        cont[..8].copy_from_slice(&64u64.to_le_bytes());
+        let data = build_v2_header(0x00, &[(0x10, &cont, 0)], None);
+        assert_eq!(
+            ObjectHeader::parse(&data, 0, 8, 8).unwrap_err(),
+            FormatError::InvalidObjectHeader("invalid continuation chunk size (0)")
+        );
+        let data = build_v2_header(0x00, &[(0x10, &cont[..8], 0)], None);
+        assert_eq!(
+            ObjectHeader::parse(&data, 0, 8, 8).unwrap_err(),
+            FormatError::InvalidObjectHeader("ran off end of input buffer while decoding")
+        );
+    }
+
+    #[test]
+    fn v2_prefix_is_checked() {
+        let data = build_v2_header(0x40, &[(0x01, &[1], 0)], None);
+        assert_eq!(
+            ObjectHeader::parse(&data, 0, 8, 8).unwrap_err(),
+            FormatError::InvalidObjectHeader("unknown object header status flag(s)")
+        );
+        // build_v2_header writes max_compact 8, min_dense 6; swap them.
+        let mut data = build_v2_header(0x10, &[(0x01, &[1], 0)], None);
+        data[6] = 6;
+        data[8] = 8;
+        let chk = crate::checksum::jenkins_lookup3(&data[..data.len() - 4]);
+        let n = data.len();
+        data[n - 4..].copy_from_slice(&chk.to_le_bytes());
+        assert_eq!(
+            ObjectHeader::parse(&data, 0, 8, 8).unwrap_err(),
+            FormatError::InvalidObjectHeader("bad object header attribute phase change values")
+        );
+    }
+
+    #[test]
+    fn v2_gap_is_allowed_only_without_nil_messages() {
+        // Three bytes after the last message: a gap (a message header is 4).
+        let mut data = build_v2_header(0x00, &[(0x01, &[1, 2, 3], 0), (0x03, &[], 0)], None);
+        // Turn the empty datatype message (4 header bytes) into a 3-byte gap
+        // by shrinking the chunk.
+        let n = data.len();
+        data.truncate(n - 5);
+        data[6] -= 1;
+        let chk = crate::checksum::jenkins_lookup3(&data);
+        data.extend_from_slice(&chk.to_le_bytes());
+        assert_eq!(
+            ObjectHeader::parse(&data, 0, 8, 8).unwrap().messages.len(),
+            1
+        );
+
+        let mut data = build_v2_header(0x00, &[(0x00, &[1, 2, 3], 0), (0x03, &[], 0)], None);
+        let n = data.len();
+        data.truncate(n - 5);
+        data[6] -= 1;
+        let chk = crate::checksum::jenkins_lookup3(&data);
+        data.extend_from_slice(&chk.to_le_bytes());
+        assert_eq!(
+            ObjectHeader::parse(&data, 0, 8, 8).unwrap_err(),
+            FormatError::InvalidObjectHeader("gap in chunk with no null messages")
+        );
+    }
+
     #[test]
     fn parse_v2_unknown_message_flags() {
         let data = build_v2_header(0x00, &[(0xF0, &[1, 2], 0x08)], None);

crates/clawhdf5-format/src/parallel_read.rs

@@ -10,7 +10,7 @@
 use crate::chunked_read::ChunkInfo;
 use crate::error::FormatError;
 use crate::filter_pipeline::FilterPipeline;
-use crate::filters::decompress_chunk_masked;
+use crate::filters::decompress_chunk_exact;
 use crate::lane_partition::{self, LaneStats, PartitionStats};
 
 /// Threshold: only use parallel decompression when chunk count exceeds this.
@@ -84,12 +84,13 @@ pub fn decompress_chunks_lane_partitioned(
                 }
                 let raw_chunk = &file_data[c_addr..c_addr + size];
 
-                let decompressed = decompress_chunk_masked(
+                let decompressed = decompress_chunk_exact(
                     raw_chunk,
                     pipeline,
                     chunk_total_bytes,
                     element_size,
                     chunk_info.filter_mask,
+                    &chunk_info.offsets,
                 )?;
 
                 stats.chunks_processed += 1;
@@ -160,12 +161,13 @@ pub fn decompress_chunks_parallel(
             }
             let raw_chunk = &file_data[c_addr..c_addr + size];
 
-            let decompressed = decompress_chunk_masked(
+            let decompressed = decompress_chunk_exact(
                 raw_chunk,
                 pipeline,
                 chunk_total_bytes,
                 element_size,
                 chunk_info.filter_mask,
+                &chunk_info.offsets,
             )?;
 
             Ok(DecompressedChunk {
@@ -204,12 +206,13 @@ pub fn decompress_chunks_sequential(
         let raw_chunk = &file_data[c_addr..c_addr + size];
 
         let decompressed = if let Some(pl) = pipeline {
-            decompress_chunk_masked(
+            decompress_chunk_exact(
                 raw_chunk,
                 pl,
                 chunk_total_bytes,
                 element_size,
                 chunk_info.filter_mask,
+                &chunk_info.offsets,
             )?
         } else {
             raw_chunk.to_vec()
@@ -218,3 +221,60 @@ pub fn decompress_chunks_sequential(
     }
     Ok(result)
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::filter_pipeline::{FILTER_SHUFFLE, FilterDescription};
+
+    /// Eight shuffled 32-byte chunks; chunk 5 is stored short when `short`.
+    fn chunks(short: bool) -> (Vec<u8>, Vec<ChunkInfo>) {
+        let mut file = Vec::new();
+        let mut infos = Vec::new();
+        for i in 0..8u64 {
+            let len = if short && i == 5 { 16 } else { 32 };
+            infos.push(ChunkInfo {
+                chunk_size: len as u32,
+                filter_mask: 0,
+                offsets: vec![i * 8],
+                address: file.len() as u64,
+            });
+            file.extend(core::iter::repeat_n(i as u8, len));
+        }
+        (file, infos)
+    }
+
+    /// Every parallel decoder refuses a chunk that decodes short, naming it.
+    #[test]
+    fn short_decoded_chunk_is_an_error() {
+        let pipeline = FilterPipeline {
+            version: 2,
+            filters: vec![FilterDescription {
+                filter_id: FILTER_SHUFFLE,
+                name: None,
+                flags: 0,
+                client_data: vec![4],
+            }],
+        };
+        let (file, good) = chunks(false);
+        assert_eq!(
+            decompress_chunks_parallel(&file, &good, &pipeline, 32, 4).unwrap()[5],
+            [5u8; 32]
+        );
+        let (file, bad) = chunks(true);
+        let errs = [
+            decompress_chunks_lane_partitioned(&file, &bad, &pipeline, 32, 4, 1, Some(3))
+                .map(|_| ())
+                .unwrap_err(),
+            decompress_chunks_parallel(&file, &bad, &pipeline, 32, 4)
+                .map(|_| ())
+                .unwrap_err(),
+            decompress_chunks_sequential(&file, &bad, Some(&pipeline), 32, 4)
+                .map(|_| ())
+                .unwrap_err(),
+        ];
+        for e in errs {
+            assert!(e.to_string().contains("[40]"), "{e}");
+        }
+    }
+}

crates/clawhdf5-format/src/partial_read.rs

@@ -22,7 +22,7 @@ use crate::data_read::extract_selection_from_buffer;
 use crate::dataspace::Dataspace;
 use crate::error::FormatError;
 use crate::filter_pipeline::FilterPipeline;
-use crate::filters::{all_filters_skipped, decompress_chunk_masked};
+use crate::filters::{all_filters_skipped, decompress_chunk_exact};
 use crate::selection::Selection;
 
 /// The smallest axis-aligned box containing every selected element, as
@@ -330,12 +330,13 @@ pub fn read_selection(
                 let decoded;
                 let data: &[u8] = match pipeline {
                     Some(pl) if !all_filters_skipped(pl, chunk.filter_mask) => {
-                        decoded = decompress_chunk_masked(
+                        decoded = decompress_chunk_exact(
                             raw,
                             pl,
                             chunk_bytes,
                             elem_size as u32,
                             chunk.filter_mask,
+                            &chunk.offsets[..rank],
                         )?;
                         &decoded
                     }

crates/clawhdf5-format/src/selection.rs

@@ -229,44 +229,47 @@ impl Selection {
     /// self-describing in length, so the count lets a caller walk a packed list
     /// of selections — as the Virtual Dataset global-heap block does).
     ///
-    /// Only the forms needed for VDS assembly are decoded: `ALL`, `NONE`, and
+    /// Decodes `ALL`, `NONE`, and hyperslabs at every version libhdf5 writes
-    /// **regular** hyperslabs serialized at **version 3** (the encoding HDF5
+    /// (1: irregular, 4-byte coordinates — the default-format encoding; 2:
-    /// 1.10+/2.0 emit). Point selections, irregular hyperslabs, and older
+    /// regular, 8-byte; 3: either, variable width). A regular hyperslab maps
-    /// hyperslab versions return an error rather than mis-decoding.
+    /// to [`Selection::Hyperslab`]; an *irregular* one (a union of blocks)
+    /// maps to a single-block hyperslab when it has one block, and otherwise to
+    /// [`Selection::Points`] listing the union in row-major order (the order
+    /// libhdf5 iterates it in). Unlimited counts/blocks decode as `u64::MAX`
+    /// (see [`SerializedSelection::decode`] for the raw form). Point
+    /// selections are refused: libhdf5 does not allow them in virtual datasets
+    /// either.
     pub fn decode_serialized(data: &[u8]) -> Result<(Selection, usize), FormatError> {
-        if data.len() < 8 {
+        let (raw, len) = SerializedSelection::decode(data)?;
-            return Err(FormatError::UnexpectedEof {
+        let sel = match raw {
-                expected: 8,
+            SerializedSelection::All => Selection::All,
-                available: data.len(),
+            SerializedSelection::None => Selection::None,
-            });
+            SerializedSelection::Regular {
+                start,
+                stride,
+                count,
+                block,
+            } => Selection::Hyperslab {
+                start,
+                stride,
+                count,
+                block,
+            },
+            SerializedSelection::Blocks { rank, starts, ends } => {
+                if starts.len() == rank {
+                    let block = starts.iter().zip(&ends).map(|(&s, &e)| e - s + 1).collect();
+                    Selection::Hyperslab {
+                        start: starts,
+                        stride: vec![1; rank],
+                        count: vec![1; rank],
+                        block,
                     }
-        let sel_type = u32::from_le_bytes([data[0], data[1], data[2], data[3]]);
-        let version = u32::from_le_bytes([data[4], data[5], data[6], data[7]]);
-
-        match sel_type {
-            // ALL / NONE: type(4) + version(4) + reserved(4) + length(4) = 16 bytes.
-            3 | 0 => {
-                if data.len() < 16 {
-                    return Err(FormatError::UnexpectedEof {
-                        expected: 16,
-                        available: data.len(),
-                    });
-                }
-                let sel = if sel_type == 3 {
-                    Selection::All
                 } else {
-                    Selection::None
+                    Selection::Points(blocks_union_coords(rank, &starts, &ends)?)
+                }
+            }
         };
-                Ok((sel, 16))
+        Ok((sel, len))
-            }
-            2 => decode_hyperslab_serialized(data, version),
-            1 => Err(FormatError::ChunkedReadError(
-                "VDS point selections are not supported".into(),
-            )),
-            _ => Err(FormatError::ChunkedReadError(
-                "unknown dataspace selection type".into(),
-            )),
-        }
     }
 
     /// Enumerate the selected element indices of a **1-D** dataspace of the
@@ -314,6 +317,11 @@ impl Selection {
                         "VDS selection rank does not match dataspace rank".into(),
                     ));
                 }
+                if count.iter().chain(block.iter()).any(|&v| v == UNLIMITED) {
+                    return Err(FormatError::ChunkedReadError(
+                        "unlimited selection must be clipped before it is enumerated".into(),
+                    ));
+                }
                 // Selected coordinates along each dimension, in order.
                 let mut per_dim: Vec<Vec<u64>> = Vec::with_capacity(rank);
                 for d in 0..rank {
@@ -400,59 +408,174 @@ impl Selection {
     }
 }
 
-/// Decode an `H5S_SEL_HYPER` selection in its serialized form. Only version-3
+/// Hyperslab count/block value meaning "unlimited" (`H5S_UNLIMITED`).
-/// **regular** hyperslabs are supported.
+pub const UNLIMITED: u64 = u64::MAX;
-fn decode_hyperslab_serialized(
+
-    data: &[u8],
+/// Largest number of elements an irregular selection is expanded to when it
-    version: u32,
+/// is converted to a point list by [`Selection::decode_serialized`].
-) -> Result<(Selection, usize), FormatError> {
+const MAX_EXPANDED_POINTS: u64 = 1 << 26;
-    if version != 3 {
+
-        return Err(FormatError::ChunkedReadError(
+/// A selection exactly as `H5S_select_serialize` stores it, before it is
-            "only version-3 hyperslab selections are supported".into(),
+/// applied to any dataspace.
-        ));
+///
+/// Unlike [`Selection`] this keeps an irregular hyperslab as its list of
+/// blocks, and a regular hyperslab's count/block may be [`UNLIMITED`] (the
+/// unlimited selections used by unlimited and "printf" virtual dataset
+/// mappings).
+#[derive(Debug, Clone, PartialEq)]
+pub enum SerializedSelection {
+    /// `H5S_SEL_ALL`.
+    All,
+    /// `H5S_SEL_NONE`.
+    None,
+    /// A regular hyperslab. `count[d]` or `block[d]` may be [`UNLIMITED`].
+    Regular {
+        start: Vec<u64>,
+        stride: Vec<u64>,
+        count: Vec<u64>,
+        block: Vec<u64>,
+    },
+    /// An irregular hyperslab: the union of `starts.len() / rank` blocks, each
+    /// given by its first (`starts`) and last (`ends`, inclusive) coordinate,
+    /// flattened block-major.
+    Blocks {
+        rank: usize,
+        starts: Vec<u64>,
+        ends: Vec<u64>,
+    },
 }
-    // type(4) ver(4) flags(1) enc_size(1) rank(4) [start,stride,count,block]*rank
+
-    if data.len() < 14 {
+fn sel_err(msg: &str) -> FormatError {
-        return Err(FormatError::UnexpectedEof {
+    FormatError::ChunkedReadError(msg.into())
-            expected: 14,
-            available: data.len(),
-        });
 }
-    let flags = data[8];
+
-    let enc_size = data[9] as usize;
+/// Bounds-checked little-endian reader over a serialized selection.
-    // Bit 0 set => regular hyperslab. Irregular hyperslabs list explicit blocks.
+struct SelReader<'a> {
-    if flags & 0x01 == 0 {
+    data: &'a [u8],
-        return Err(FormatError::ChunkedReadError(
+    pos: usize,
-            "irregular VDS hyperslab selections are not supported".into(),
-        ));
 }
-    if enc_size != 2 && enc_size != 4 && enc_size != 8 {
+
-        return Err(FormatError::ChunkedReadError(
+impl SelReader<'_> {
-            "unsupported hyperslab coordinate encoding size".into(),
+    fn take(&mut self, n: usize) -> Result<&[u8], FormatError> {
-        ));
+        let end = self.pos.checked_add(n).filter(|&e| e <= self.data.len());
+        let end = end.ok_or(FormatError::UnexpectedEof {
+            expected: self.pos.saturating_add(n),
+            available: self.data.len(),
+        })?;
+        let s = &self.data[self.pos..end];
+        self.pos = end;
+        Ok(s)
     }
-    let rank = u32::from_le_bytes([data[10], data[11], data[12], data[13]]) as usize;
+
-    // HDF5 caps dataspace rank at 32 (H5S_MAX_RANK). Reject anything larger so a
+    fn uint(&mut self, size: usize) -> Result<u64, FormatError> {
-    // corrupt rank can't drive a huge allocation or read loop.
+        let bytes = self.take(size)?;
-    if rank > 32 {
+        Ok(bytes
-        return Err(FormatError::ChunkedReadError(
+            .iter()
-            "hyperslab selection rank exceeds maximum (32)".into(),
+            .enumerate()
-        ));
+            .fold(0u64, |v, (i, &b)| v | (b as u64) << (i * 8)))
     }
-    let mut pos = 14;
+
-    let read_coord = |data: &[u8], pos: usize| -> Result<u64, FormatError> {
+    fn remaining(&self) -> usize {
-        if pos + enc_size > data.len() {
+        self.data.len() - self.pos
-            return Err(FormatError::UnexpectedEof {
-                expected: pos + enc_size,
-                available: data.len(),
-            });
     }
-        let mut v = 0u64;
-        for (i, &b) in data[pos..pos + enc_size].iter().enumerate() {
-            v |= (b as u64) << (i * 8);
 }
-        Ok(v)
+
+impl SerializedSelection {
+    /// Decode a serialized selection, returning it and the number of bytes it
+    /// occupies. Mirrors libhdf5's `H5S_select_deserialize`: `ALL`/`NONE` and
+    /// hyperslab versions 1-3 are decoded; point selections (which libhdf5
+    /// refuses in virtual datasets) and malformed input are errors.
+    pub fn decode(data: &[u8]) -> Result<(SerializedSelection, usize), FormatError> {
+        let mut r = SelReader { data, pos: 0 };
+        let sel_type = r.uint(4)?;
+        let version = r.uint(4)?;
+        match sel_type {
+            // ALL / NONE: type(4) + version(4) + reserved(4) + length(4).
+            0 | 3 => {
+                r.take(8)?;
+                let sel = if sel_type == 3 {
+                    SerializedSelection::All
+                } else {
+                    SerializedSelection::None
                 };
+                Ok((sel, r.pos))
+            }
+            2 => {
+                let sel = decode_hyperslab(&mut r, version)?;
+                Ok((sel, r.pos))
+            }
+            1 => Err(sel_err(
+                "VDS point selections are not supported (libhdf5 rejects them too)",
+            )),
+            _ => Err(sel_err("unknown dataspace selection type")),
+        }
+    }
+
+    /// The single dimension in which this selection is unlimited, if any.
+    pub fn unlimited_dim(&self) -> Option<usize> {
+        match self {
+            SerializedSelection::Regular { count, block, .. } => count
+                .iter()
+                .zip(block)
+                .position(|(&c, &b)| c == UNLIMITED || b == UNLIMITED),
+            _ => None,
+        }
+    }
+
+    /// The rank the selection was serialized with (`None` for ALL/NONE, which
+    /// carry no rank).
+    pub fn rank(&self) -> Option<usize> {
+        match self {
+            SerializedSelection::Regular { start, .. } => Some(start.len()),
+            SerializedSelection::Blocks { rank, .. } => Some(*rank),
+            _ => None,
+        }
+    }
+}
+
+/// `H5S__hyper_deserialize`: after the type and version words.
+fn decode_hyperslab(r: &mut SelReader, version: u64) -> Result<SerializedSelection, FormatError> {
+    const REGULAR: u8 = 0x01;
+    let (flags, enc_size) = match version {
+        // v1: reserved(4) + length(4), always irregular, 4-byte coordinates.
+        1 => {
+            r.take(8)?;
+            (0u8, 4usize)
+        }
+        // v2: flags(1) + length(4), 8-byte coordinates.
+        2 => {
+            let flags = r.take(1)?[0];
+            r.take(4)?;
+            (flags, 8)
+        }
+        // v3: flags(1) + encoding size(1).
+        3 => {
+            let flags = r.take(1)?[0];
+            let enc = r.take(1)?[0] as usize;
+            (flags, enc)
+        }
+        _ => return Err(sel_err("unsupported hyperslab selection version")),
+    };
+    if flags & !REGULAR != 0 {
+        return Err(sel_err("unknown hyperslab selection flags"));
+    }
+    if !matches!(enc_size, 2 | 4 | 8) {
+        return Err(sel_err("unsupported hyperslab coordinate encoding size"));
+    }
+    let rank = r.uint(4)? as usize;
+    // HDF5 caps dataspace rank at 32 (H5S_MAX_RANK). Reject anything else so a
+    // corrupt rank can't drive a huge allocation or read loop.
+    if rank == 0 || rank > 32 {
+        return Err(sel_err("hyperslab selection rank must be 1..=32"));
+    }
+    // The all-ones value of the encoding width means "unlimited".
+    let unlim_raw = if enc_size == 8 {
+        u64::MAX
+    } else {
+        (1u64 << (enc_size * 8)) - 1
+    };
+
+    if flags & REGULAR != 0 {
         let (mut start, mut stride, mut count, mut block) = (
             Vec::with_capacity(rank),
             Vec::with_capacity(rank),
@@ -460,24 +583,104 @@ fn decode_hyperslab_serialized(
             Vec::with_capacity(rank),
         );
         for _ in 0..rank {
-        start.push(read_coord(data, pos)?);
+            start.push(r.uint(enc_size)?);
-        pos += enc_size;
+            stride.push(r.uint(enc_size)?);
-        stride.push(read_coord(data, pos)?);
+            let c = r.uint(enc_size)?;
-        pos += enc_size;
+            count.push(if c == unlim_raw { UNLIMITED } else { c });
-        count.push(read_coord(data, pos)?);
+            let b = r.uint(enc_size)?;
-        pos += enc_size;
+            block.push(if b == unlim_raw { UNLIMITED } else { b });
-        block.push(read_coord(data, pos)?);
-        pos += enc_size;
         }
-    Ok((
+        let unlimited = count
-        Selection::Hyperslab {
+            .iter()
+            .zip(&block)
+            .filter(|&(&c, &b)| c == UNLIMITED || b == UNLIMITED)
+            .count();
+        if unlimited > 1 {
+            return Err(sel_err(
+                "hyperslab selection is unlimited in more than one dimension",
+            ));
+        }
+        for d in 0..rank {
+            // Overlapping blocks are not a valid regular hyperslab.
+            if count[d] > 1 && block[d] != UNLIMITED && block[d] > stride[d] {
+                return Err(sel_err("regular hyperslab blocks overlap"));
+            }
+        }
+        return Ok(SerializedSelection::Regular {
             start,
             stride,
             count,
             block,
-        },
+        });
-        pos,
+    }
-    ))
+
+    // Irregular: number of blocks, then each block's start and end corners.
+    let nblocks = r.uint(enc_size)?;
+    let per_block = (rank * 2 * enc_size) as u64;
+    // Untrusted count: it must fit in what is left of the buffer.
+    if nblocks
+        .checked_mul(per_block)
+        .is_none_or(|need| need > r.remaining() as u64)
+    {
+        return Err(FormatError::UnexpectedEof {
+            expected: r
+                .pos
+                .saturating_add(nblocks.saturating_mul(per_block) as usize),
+            available: r.data.len(),
+        });
+    }
+    let n = nblocks as usize * rank;
+    let (mut starts, mut ends) = (Vec::with_capacity(n), Vec::with_capacity(n));
+    for _ in 0..nblocks {
+        for _ in 0..rank {
+            starts.push(r.uint(enc_size)?);
+        }
+        for _ in 0..rank {
+            ends.push(r.uint(enc_size)?);
+        }
+    }
+    if starts.iter().zip(&ends).any(|(s, e)| e < s) {
+        return Err(sel_err("hyperslab block ends before it starts"));
+    }
+    Ok(SerializedSelection::Blocks { rank, starts, ends })
+}
+
+/// The coordinates of the union of the given blocks, in row-major order.
+fn blocks_union_coords(
+    rank: usize,
+    starts: &[u64],
+    ends: &[u64],
+) -> Result<Vec<Vec<u64>>, FormatError> {
+    let mut total = 0u64;
+    for (s, e) in starts.chunks_exact(rank).zip(ends.chunks_exact(rank)) {
+        let vol = s
+            .iter()
+            .zip(e)
+            .try_fold(1u64, |acc, (&s, &e)| acc.checked_mul(e - s + 1));
+        total = vol
+            .and_then(|v| total.checked_add(v))
+            .filter(|&t| t <= MAX_EXPANDED_POINTS)
+            .ok_or_else(|| sel_err("irregular hyperslab selection is too large to expand"))?;
+    }
+    let mut out = Vec::with_capacity(total as usize);
+    for (s, e) in starts.chunks_exact(rank).zip(ends.chunks_exact(rank)) {
+        let mut cur = s.to_vec();
+        'block: loop {
+            out.push(cur.clone());
+            for d in (0..rank).rev() {
+                if cur[d] < e[d] {
+                    cur[d] += 1;
+                    continue 'block;
+                }
+                cur[d] = s[d];
+            }
+            break;
+        }
+    }
+    // Lexicographic order of coordinates is row-major order.
+    out.sort_unstable();
+    out.dedup();
+    Ok(out)
 }
 
 // ---------------------------------------------------------------------------
@@ -642,11 +845,100 @@ mod tests {
     }
 
     #[test]
-    fn decode_irregular_hyperslab_rejected() {
+    fn decode_truncated_irregular_hyperslab_is_error() {
+        // Irregular, rank 1, but the block count is missing.
         let bytes = [0x02u8, 0, 0, 0, 0x03, 0, 0, 0, 0x00, 0x02, 0x01, 0, 0, 0];
         assert!(Selection::decode_serialized(&bytes).is_err());
     }
 
+    /// Version 1 as libhdf5 writes it for the default (earliest) format bounds:
+    /// type, version, reserved(4), length(4), rank(4), nblocks(4), then each
+    /// block's start and inclusive end corner as 4-byte values.
+    fn v1_blocks(rank: u32, blocks: &[(&[u32], &[u32])]) -> Vec<u8> {
+        let mut b = Vec::new();
+        for w in [2u32, 1, 0, 0, rank, blocks.len() as u32] {
+            b.extend_from_slice(&w.to_le_bytes());
+        }
+        for (s, e) in blocks {
+            for v in s.iter().chain(e.iter()) {
+                b.extend_from_slice(&v.to_le_bytes());
+            }
+        }
+        b
+    }
+
+    #[test]
+    fn decode_v1_irregular_single_block() {
+        // Exactly what h5py/HDF5 2.0 writes for `[0:4]` with default libver.
+        let bytes = v1_blocks(1, &[(&[0], &[3])]);
+        let (sel, used) = Selection::decode_serialized(&bytes).unwrap();
+        assert_eq!(used, bytes.len());
+        assert_eq!(sel.iter_linear_1d(8).unwrap(), vec![0, 1, 2, 3]);
+    }
+
+    #[test]
+    fn decode_v1_irregular_union_is_row_major() {
+        // Blocks given out of order and overlapping still enumerate once each,
+        // in row-major order (libhdf5 iterates the union, not the list).
+        let bytes = v1_blocks(2, &[(&[1, 0], &[1, 1]), (&[0, 2], &[1, 2])]);
+        let (sel, used) = Selection::decode_serialized(&bytes).unwrap();
+        assert_eq!(used, bytes.len());
+        // (0,2) (1,0) (1,1) (1,2) in a 2x3 space.
+        assert_eq!(sel.iter_linear(&[2, 3]).unwrap(), vec![2, 3, 4, 5]);
+    }
+
+    #[test]
+    fn decode_v2_regular_with_unlimited_count() {
+        // v2: flags(1) + length(4), then 8-byte start/stride/count/block.
+        let mut b = Vec::new();
+        b.extend_from_slice(&2u32.to_le_bytes());
+        b.extend_from_slice(&2u32.to_le_bytes());
+        b.push(0x01);
+        b.extend_from_slice(&36u32.to_le_bytes());
+        b.extend_from_slice(&1u32.to_le_bytes());
+        for v in [0u64, 10, u64::MAX, 10] {
+            b.extend_from_slice(&v.to_le_bytes());
+        }
+        let (raw, used) = SerializedSelection::decode(&b).unwrap();
+        assert_eq!(used, b.len());
+        assert_eq!(raw.unlimited_dim(), Some(0));
+        assert_eq!(
+            raw,
+            SerializedSelection::Regular {
+                start: vec![0],
+                stride: vec![10],
+                count: vec![UNLIMITED],
+                block: vec![10],
+            }
+        );
+        // An unclipped unlimited selection cannot be enumerated.
+        let (sel, _) = Selection::decode_serialized(&b).unwrap();
+        assert!(sel.iter_linear_1d(100).is_err());
+    }
+
+    #[test]
+    fn decode_v3_two_byte_all_ones_is_unlimited() {
+        let bytes = [
+            0x02, 0, 0, 0, 0x03, 0, 0, 0, 0x01, 0x02, 0x01, 0, 0, 0, //
+            0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0xFF, 0xFF,
+        ];
+        let (raw, _) = SerializedSelection::decode(&bytes).unwrap();
+        assert_eq!(raw.unlimited_dim(), Some(0));
+    }
+
+    #[test]
+    fn decode_irregular_block_count_beyond_buffer_is_error() {
+        let mut b = v1_blocks(1, &[(&[0], &[3])]);
+        b[20..24].copy_from_slice(&u32::MAX.to_le_bytes());
+        assert!(Selection::decode_serialized(&b).is_err());
+    }
+
+    #[test]
+    fn decode_point_selection_is_refused() {
+        let bytes = [1u8, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0];
+        assert!(Selection::decode_serialized(&bytes).is_err());
+    }
+
     #[test]
     fn iter_linear_2d_block_row_major() {
         // A 2x2 block at the top-left of a 4x4 space => linear 0,1,4,5.

crates/clawhdf5-format/src/shared_message.rs

@@ -154,13 +154,29 @@ pub fn is_shared(msg_flags: u8) -> bool {
 ///
 /// When the shared flag is set on a message, the data contains a reference
 /// instead of the actual message content.
+///
+/// Assumes the file's length size equals its offset size, which only matters
+/// for version-1 references; use [`parse_shared_ref_sized`] when the
+/// superblock's length size is known.
 pub fn parse_shared_ref(data: &[u8], offset_size: u8) -> Result<SharedMessageRef, FormatError> {
+    parse_shared_ref_sized(data, offset_size, offset_size)
+}
+
+/// [`parse_shared_ref`] with the superblock's length size, which locates the
+/// object header address in a version-1 reference.
+pub fn parse_shared_ref_sized(
+    data: &[u8],
+    offset_size: u8,
+    length_size: u8,
+) -> Result<SharedMessageRef, FormatError> {
     ensure_len(data, 0, 2)?;
     let version = data[0];
     let ref_type = data[1];
 
     // Layouts (HDF5 spec IV.A.2 "Shared Message", and libhdf5's decoder):
-    //   v1: version, type, reserved(6), address     — always "committed"
+    //   v1: version, type, reserved(6), then an old-style symbol table
+    //       entry: link-name offset(length_size), object header address,
+    //       cache type(4), reserved(4), scratch(16)  — always "committed"
     //   v2: version, type, address                  — always "committed"
     //   v3: version, type, then a fractal-heap ID if type == SOHM, otherwise
     //       an address
@@ -177,7 +193,7 @@ pub fn parse_shared_ref(data: &[u8], offset_size: u8) -> Result<SharedMessageRef
         })
     };
     match version {
-        1 => address_at(2 + 6),
+        1 => address_at(2 + 6 + length_size as usize),
         2 => address_at(2),
         3 if ref_type == SHARE_TYPE_SOHM => {
             ensure_len(data, 2, FHEAP_ID_LEN)?;
@@ -434,7 +450,7 @@ pub fn message_data_with_sohm<'a>(
     if !is_shared(msg.flags) {
         return Ok(Cow::Borrowed(&msg.data));
     }
-    let shared_ref = parse_shared_ref(&msg.data, offset_size)?;
+    let shared_ref = parse_shared_ref_sized(&msg.data, offset_size, length_size)?;
     let table = if shared_ref.heap_id.is_some() {
         load_sohm_table(file_data, offset_size, length_size)?
     } else {
@@ -514,7 +530,7 @@ pub fn message_data<'a>(
     if !is_shared(msg.flags) {
         return Ok(Cow::Borrowed(&msg.data));
     }
-    let shared_ref = parse_shared_ref(&msg.data, offset_size)?;
+    let shared_ref = parse_shared_ref_sized(&msg.data, offset_size, length_size)?;
     resolve_shared_message(
         file_data,
         &shared_ref,
@@ -529,7 +545,8 @@ pub fn message_data<'a>(
 ///
 /// For type 1/3 (shared in another object header), reads the target object header
 /// and finds the message of the specified type.
-/// For type 2 (SOHM), uses the fractal heap from the SOHM table.
+/// For type 2 (SOHM), uses the fractal heap from the file's SOHM table,
+/// loaded from the superblock extension on demand.
 pub fn resolve_shared_message(
     file_data: &[u8],
     shared_ref: &SharedMessageRef,
@@ -537,13 +554,18 @@ pub fn resolve_shared_message(
     offset_size: u8,
     length_size: u8,
 ) -> Result<Vec<u8>, FormatError> {
+    let table = if shared_ref.heap_id.is_some() {
+        load_sohm_table(file_data, offset_size, length_size)?
+    } else {
+        None
+    };
     resolve_shared_message_with_sohm(
         file_data,
         shared_ref,
         target_msg_type,
         offset_size,
         length_size,
-        None,
+        table.as_ref(),
     )
 }
 
@@ -649,15 +671,26 @@ mod tests {
 
     #[test]
     fn parse_v1_ref() {
-        let mut data = Vec::new();
+        // Datatype message of `/group1/dset2` in HDF5's `tcompound.h5`
-        data.push(1); // version
+        // (written in 2000): version 1, six reserved bytes, then an old-style
-        data.push(0); // type
+        // symbol table entry — link-name offset 0x10, object header address
-        data.extend_from_slice(&[0u8; 6]); // reserved
+        // 0x590 (the committed datatype `/type1`), cache type, reserved and
-        data.extend_from_slice(&0x5678u64.to_le_bytes());
+        // scratch.
+        let mut data = vec![1, 0, 0, 0, 0, 0, 0, 0];
+        data.extend_from_slice(&0x10u64.to_le_bytes());
+        data.extend_from_slice(&0x590u64.to_le_bytes());
+        data.extend_from_slice(&[0; 24]);
 
-        let shared = parse_shared_ref(&data, 8).unwrap();
+        let shared = parse_shared_ref_sized(&data, 8, 8).unwrap();
         assert_eq!(shared.version, 1);
-        assert_eq!(shared.object_header_address, Some(0x5678));
+        assert_eq!(shared.object_header_address, Some(0x590));
+
+        // The name offset is a length: 4 bytes here, then an 8-byte address.
+        let mut data = vec![1, 0, 0, 0, 0, 0, 0, 0];
+        data.extend_from_slice(&0x10u32.to_le_bytes());
+        data.extend_from_slice(&0x590u64.to_le_bytes());
+        let shared = parse_shared_ref_sized(&data, 8, 4).unwrap();
+        assert_eq!(shared.object_header_address, Some(0x590));
     }
 
     #[test]

crates/clawhdf5-format/src/signature.rs

@@ -11,6 +11,16 @@ pub const HDF5_SIGNATURE: [u8; 8] = [0x89, b'H', b'D', b'F', b'\r', b'\n', 0x1A,
 /// (powers of two starting at 512, plus offset 0).
 ///
 /// Returns the byte offset where the signature was found.
+///
+/// A non-zero offset means the file starts with a *user block*, and every
+/// address inside the file is relative to the superblock's position, not to
+/// byte 0 (libhdf5 uses the signature's position as the base address even
+/// when the stored base-address field disagrees). The parsers in this crate
+/// take addresses as indices into `file_data`, so they must be handed the
+/// bytes from the signature on — use [`split_user_block`]. [`Superblock::parse`]
+/// refuses a non-zero offset for this reason.
+///
+/// [`Superblock::parse`]: crate::superblock::Superblock::parse
 pub fn find_signature(data: &[u8]) -> Result<usize, FormatError> {
     // Check offset 0
     if data.len() >= 8 && data[..8] == HDF5_SIGNATURE {
@@ -29,6 +39,17 @@ pub fn find_signature(data: &[u8]) -> Result<usize, FormatError> {
     Err(FormatError::SignatureNotFound)
 }
 
+/// Split a file into its user block and its HDF5 bytes.
+///
+/// Returns `(user_block, hdf5)`: `user_block` is everything before the
+/// superblock signature (empty for most files) and `hdf5` is the rest, in
+/// which every HDF5 address is a plain index. Pass `hdf5` as `file_data` to
+/// every parser in this crate, and parse the superblock at offset 0 of it.
+pub fn split_user_block(data: &[u8]) -> Result<(&[u8], &[u8]), FormatError> {
+    let offset = find_signature(data)?;
+    Ok(data.split_at(offset))
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -88,6 +109,21 @@ mod tests {
         assert_eq!(find_signature(&data), Err(FormatError::SignatureNotFound));
     }
 
+    #[test]
+    fn split_user_block_rebases_at_the_signature() {
+        let mut data = vec![7u8; 1024];
+        data[512..520].copy_from_slice(&HDF5_SIGNATURE);
+        let (ub, hdf5) = split_user_block(&data).unwrap();
+        assert_eq!(ub.len(), 512);
+        assert_eq!(hdf5.len(), 512);
+        assert_eq!(&hdf5[..8], &HDF5_SIGNATURE);
+
+        data[..8].copy_from_slice(&HDF5_SIGNATURE);
+        let (ub, hdf5) = split_user_block(&data).unwrap();
+        assert!(ub.is_empty());
+        assert_eq!(hdf5.len(), 1024);
+    }
+
     #[test]
     fn signature_prefers_earliest() {
         // Signature at both 0 and 512, should return 0

crates/clawhdf5-format/src/superblock.rs

@@ -100,6 +100,42 @@ pub mod swmr_flags {
 }
 
 impl Superblock {
+    /// Where the HDF5 data ends, relative to the superblock, for a file of
+    /// `file_len` bytes whose superblock is at `user_block` (both counted
+    /// from the start of the file), with libhdf5's truncation check
+    /// (`H5F__super_read`).
+    ///
+    /// The superblock records the end of the file's data as an absolute
+    /// address. A file shorter than that was truncated, and libhdf5 refuses
+    /// to open it ("truncated file"); so does this, with
+    /// [`FormatError::TruncatedFile`]. Bytes past that address are not part
+    /// of the file: libhdf5 fails any read of them ("addr overflow" /
+    /// "address plus size exceeds file eoa"), so a reader should parse only
+    /// the data up to the returned end. As libhdf5 does for a SWMR reader,
+    /// the check is skipped for a version-3 superblock whose writer is still
+    /// writing it in SWMR mode (it extends the file as it goes); the data
+    /// then ends at the end of the file.
+    ///
+    /// When the superblock's recorded base address differs from where the
+    /// superblock actually is (a user block added or removed after the file
+    /// was written), libhdf5 moves the recorded end of file by the same
+    /// amount, and so does this.
+    pub fn data_end(&self, user_block: u64, file_len: u64) -> Result<u64, FormatError> {
+        let eof =
+            i128::from(self.eof_address) - i128::from(self.base_address) + i128::from(user_block);
+        if eof < 0 || eof > i128::from(file_len) {
+            if self.version >= 3 && self.is_swmr_write() {
+                return Ok(file_len.saturating_sub(user_block));
+            }
+            return Err(FormatError::TruncatedFile {
+                stored_eof: u64::try_from(eof).unwrap_or(self.eof_address),
+                actual_len: file_len,
+            });
+        }
+        // 0 <= eof <= file_len, so it fits a u64.
+        Ok((eof as u64).saturating_sub(user_block))
+    }
+
     /// Whether the file was opened with write access when the superblock was written.
     pub fn is_write_access(&self) -> bool {
         self.consistency_flags & swmr_flags::WRITE_ACCESS != 0
@@ -174,8 +210,18 @@ impl Superblock {
 
     /// Parse a superblock from `data` starting at `signature_offset`.
     ///
-    /// The signature must be present at the given offset.
+    /// The signature must be present at the given offset, and that offset
+    /// must be 0: every address in an HDF5 file is relative to the
+    /// superblock, so when a file has a user block (signature at 512, 1024,
+    /// …) the caller must pass the bytes from the signature on — see
+    /// [`crate::signature::split_user_block`] — and use that slice as
+    /// `file_data` everywhere. A non-zero offset is refused with
+    /// [`FormatError::UserBlockNotStripped`] because the addresses in the
+    /// returned superblock would otherwise be applied to the wrong bytes.
     pub fn parse(data: &[u8], signature_offset: usize) -> Result<Superblock, FormatError> {
+        if signature_offset != 0 {
+            return Err(FormatError::UserBlockNotStripped(signature_offset as u64));
+        }
         let d = data
             .get(signature_offset..)
             .ok_or(FormatError::UnexpectedEof {
@@ -527,6 +573,30 @@ mod tests {
         buf
     }
 
+    #[test]
+    fn data_end_refuses_truncated_files_like_libhdf5() {
+        // build_v2_bytes records base 0, end of file 2048.
+        let sb = Superblock::parse(&build_v2_bytes(8, 2), 0).unwrap();
+        assert_eq!(sb.data_end(0, 2048), Ok(2048));
+        // Bytes past the recorded end are not part of the file.
+        assert_eq!(sb.data_end(0, 4096), Ok(2048));
+        assert_eq!(
+            sb.data_end(0, 2047),
+            Err(FormatError::TruncatedFile {
+                stored_eof: 2048,
+                actual_len: 2047
+            })
+        );
+        // A user block added in front after the file was written (the
+        // recorded base address is still 0): the end moves with it.
+        assert_eq!(sb.data_end(512, 2560), Ok(2048));
+        assert!(sb.data_end(512, 2559).is_err());
+        // A v3 superblock still being written in SWMR mode is not checked.
+        let mut swmr = Superblock::parse(&build_v2_bytes(8, 3), 0).unwrap();
+        swmr.consistency_flags = swmr_flags::WRITE_ACCESS | swmr_flags::SWMR_WRITE;
+        assert_eq!(swmr.data_end(0, 1000), Ok(1000));
+    }
+
     #[test]
     fn parse_v0_8byte_offsets() {
         let data = build_v0_bytes(8);
@@ -676,7 +746,16 @@ mod tests {
         let mut data = vec![0u8; 1024];
         let v0 = build_v0_bytes(8);
         data[512..512 + v0.len()].copy_from_slice(&v0);
-        let sb = Superblock::parse(&data, 512).unwrap();
+        // Addresses are relative to the superblock, so parsing in place
+        // (where they would be applied to the whole buffer) is refused...
+        assert_eq!(
+            Superblock::parse(&data, 512),
+            Err(FormatError::UserBlockNotStripped(512))
+        );
+        // ...and the caller parses the bytes from the signature on.
+        let (ub, hdf5) = crate::signature::split_user_block(&data).unwrap();
+        assert_eq!(ub.len(), 512);
+        let sb = Superblock::parse(hdf5, 0).unwrap();
         assert_eq!(sb.version, 0);
         assert_eq!(sb.root_group_address, 96);
     }

crates/clawhdf5-format/src/test_fuzz.rs

@@ -0,0 +1,149 @@
+//! Mutation fuzzing for the filter decoders (tests only).
+//!
+//! A decoder fed a random or mutated frame may fail, but must not panic —
+//! tests build with overflow checks and debug assertions, so an unchecked
+//! subtraction, multiplication or shift on a header field, or an
+//! out-of-range slice, fails the test — and must not return more than its
+//! output limit.
+
+#[cfg(not(feature = "std"))]
+extern crate alloc;
+#[cfg(not(feature = "std"))]
+use alloc::vec::Vec;
+
+use crate::error::FormatError;
+
+/// xorshift64*: deterministic, so a failure reproduces.
+pub(crate) struct Rng(u64);
+
+impl Rng {
+    pub(crate) fn new(seed: u64) -> Rng {
+        Rng(seed.max(1))
+    }
+
+    pub(crate) fn next_u64(&mut self) -> u64 {
+        let mut x = self.0;
+        x ^= x >> 12;
+        x ^= x << 25;
+        x ^= x >> 27;
+        self.0 = x;
+        x.wrapping_mul(0x2545_F491_4F6C_DD1D)
+    }
+
+    /// Uniform in `0..n` (`n` > 0).
+    pub(crate) fn below(&mut self, n: usize) -> usize {
+        (self.next_u64() % n as u64) as usize
+    }
+
+    pub(crate) fn bytes(&mut self, n: usize) -> Vec<u8> {
+        (0..n).map(|_| self.next_u64() as u8).collect()
+    }
+
+    /// A u32 that tends to hit edge cases in size and offset fields.
+    fn interesting_u32(&mut self, len: usize) -> u32 {
+        match self.below(10) {
+            0 => 0,
+            1 => 1,
+            2 => self.below(20) as u32,
+            3 => 15 + self.below(3) as u32,
+            4 => u32::MAX - self.below(16) as u32,
+            5 => 1 << self.below(32),
+            6 => (len as u32)
+                .wrapping_add(self.below(9) as u32)
+                .wrapping_sub(4),
+            7 => i32::MAX as u32,
+            _ => self.next_u64() as u32,
+        }
+    }
+}
+
+/// One to four random edits of `seed`.
+pub(crate) fn mutate(rng: &mut Rng, seed: &[u8]) -> Vec<u8> {
+    let mut v = seed.to_vec();
+    for _ in 0..1 + rng.below(4) {
+        let len = v.len();
+        match rng.below(9) {
+            0 if len > 0 => {
+                let i = rng.below(len);
+                v[i] ^= 1 << rng.below(8);
+            }
+            1 if len > 0 => {
+                let i = rng.below(len);
+                v[i] = rng.next_u64() as u8;
+            }
+            2 if len > 0 => {
+                let i = rng.below(len);
+                v[i] = [0, 0xff, 0x7f, 0x80, 0x20, 0x1f][rng.below(6)];
+            }
+            // A size or offset field: little- or big-endian, anywhere, but
+            // most often in the first 32 bytes where headers live.
+            3 | 4 if len >= 4 => {
+                let span = if rng.below(2) == 0 { len.min(32) } else { len };
+                let i = rng.below(span - 3);
+                let x = rng.interesting_u32(len);
+                let b = if rng.below(2) == 0 {
+                    x.to_le_bytes()
+                } else {
+                    x.to_be_bytes()
+                };
+                v[i..i + 4].copy_from_slice(&b);
+            }
+            5 if len > 0 => v.truncate(rng.below(len)),
+            6 => {
+                let n = 1 + rng.below(64);
+                let extra = rng.bytes(n);
+                v.extend_from_slice(&extra);
+            }
+            7 if len > 1 => {
+                let a = rng.below(len);
+                let b = a + rng.below(len - a);
+                let copy = v[a..b].to_vec();
+                let at = rng.below(len);
+                v.splice(at..at, copy);
+            }
+            _ if len > 0 => {
+                let i = rng.below(len);
+                v[i] = v[i].wrapping_add(1 + rng.below(3) as u8);
+            }
+            _ => v.push(rng.next_u64() as u8),
+        }
+    }
+    v
+}
+
+/// Feed `iters` inputs to `decode`: mostly mutations of `seeds`, some pure
+/// noise and some truncated seeds. Asserts only "no panic, output within
+/// `limit`".
+pub(crate) fn fuzz_decoder(
+    seed: u64,
+    seeds: &[Vec<u8>],
+    iters: usize,
+    limit: usize,
+    mut decode: impl FnMut(&[u8]) -> Result<Vec<u8>, FormatError>,
+) {
+    assert!(!seeds.is_empty());
+    let mut rng = Rng::new(seed);
+    for s in seeds {
+        // The seeds themselves must be valid, or the fuzz explores nothing.
+        decode(s).expect("seed frame must decode");
+    }
+    for _ in 0..iters {
+        let input = match rng.below(16) {
+            0 => {
+                let n = rng.below(96);
+                rng.bytes(n)
+            }
+            1 => {
+                let s = &seeds[rng.below(seeds.len())];
+                s[..rng.below(s.len() + 1)].to_vec()
+            }
+            _ => {
+                let s = &seeds[rng.below(seeds.len())];
+                mutate(&mut rng, s)
+            }
+        };
+        if let Ok(out) = decode(&input) {
+            assert!(out.len() <= limit, "decoded {} > limit {limit}", out.len());
+        }
+    }
+}

crates/clawhdf5-format/src/type_builders.rs

@@ -731,6 +731,61 @@ impl DatasetBuilder {
         self
     }
 
+    /// Compress with a plugin filter ([`PluginFilter`]), in the format the
+    /// libhdf5 plugin reads (h5py, hdf5plugin). Implies chunked storage.
+    /// Each filter needs its cargo feature (`lzf`, ...); writing fails with
+    /// `UnsupportedFilter` without it.
+    ///
+    /// [`PluginFilter`]: crate::chunked_write::PluginFilter
+    pub fn with_plugin_filter(&mut self, filter: crate::chunked_write::PluginFilter) -> &mut Self {
+        self.chunk_options.plugin = Some(filter);
+        self
+    }
+
+    /// Enable LZF compression (filter 32000) — h5py's built-in
+    /// `compression="lzf"`. Implies chunked storage; shuffle is applied
+    /// first unless `.without_shuffle()`. Requires the `lzf` cargo feature.
+    pub fn with_lzf(&mut self) -> &mut Self {
+        self.with_plugin_filter(crate::chunked_write::PluginFilter::Lzf)
+    }
+
+    /// Enable bitshuffle (filter 32008) with `compression` after the bit
+    /// transpose, in bitshuffle's default block size. Implies chunked
+    /// storage; no byte shuffle is added. Requires the `bitshuffle` cargo
+    /// feature.
+    pub fn with_bitshuffle(
+        &mut self,
+        compression: crate::chunked_write::BitshuffleCompression,
+    ) -> &mut Self {
+        self.with_plugin_filter(crate::chunked_write::PluginFilter::Bitshuffle {
+            block_size: 0,
+            compression,
+        })
+    }
+
+    /// Enable bzip2 (filter 307) at block size `level` (1-9). Implies
+    /// chunked storage; shuffle is applied first unless
+    /// `.without_shuffle()`. Requires the `bzip2` cargo feature.
+    pub fn with_bzip2(&mut self, level: u32) -> &mut Self {
+        self.with_plugin_filter(crate::chunked_write::PluginFilter::Bzip2 { level })
+    }
+
+    /// Enable Blosc (filter 32001) with `codec` at `level` (0-9) after
+    /// `shuffle`. Implies chunked storage; no extra HDF5 shuffle is added.
+    /// Requires the `blosc` cargo feature.
+    pub fn with_blosc(
+        &mut self,
+        codec: crate::chunked_write::BloscCodec,
+        level: u32,
+        shuffle: crate::chunked_write::BloscShuffle,
+    ) -> &mut Self {
+        self.with_plugin_filter(crate::chunked_write::PluginFilter::Blosc {
+            codec,
+            level,
+            shuffle,
+        })
+    }
+
     /// Enable Pcodec lossless numerical compression (private clawhdf5 filter
     /// ID 480).
     ///

crates/clawhdf5-format/tests/fixtures/legacy/README.md

@@ -0,0 +1,13 @@
+# Legacy (HDF5 1.4/1.6-era) fixtures
+
+Unmodified copies of the HDF Group's own test files from
+https://github.com/HDFGroup/hdf5 at a3cf1ea82cc7a66e50029a688121e1b105a7ce88
+(BSD-style license, see that repository's `LICENSE`). Current libraries cannot
+write these structures, so they are kept as files.
+
+| File | Upstream path | Exercises |
+|---|---|---|
+| `deflate.h5` | `test/testfiles/deflate.h5` | Data Layout message v1, chunked + deflate (v1 B-tree index) |
+| `h5ex_g_iterate.h5` | `HDF5Examples/C/H5G/h5ex_g_iterate.h5` | Data Layout message v2, contiguous; an unallocated dataset |
+| `tarrold.h5` | `test/testfiles/tarrold.h5` | Compound datatype v1 members with legacy array dimensions |
+| `tcompound.h5` | `tools/test/testfiles/tcompound.h5` | Version-1 shared messages (committed datatypes); compound v1 array members with data |

crates/clawhdf5-format/tests/integration_test.rs

@@ -83,6 +83,45 @@ fn read_chunked_dataset(file_data: &[u8], dataset_path: &str) -> (Vec<u8>, Datat
     (raw, datatype, dataspace)
 }
 
+/// Helper: read a virtual dataset with `vds::read_virtual_dataset`, giving it
+/// the dataset's own fill value (same-file sources only).
+fn read_virtual_fixture(file_data: &[u8], path: &str) -> (Vec<u8>, Datatype) {
+    let sig = find_signature(file_data).unwrap();
+    let sb = Superblock::parse(file_data, sig).unwrap();
+    let addr = resolve_path_any(file_data, &sb, path).unwrap();
+    let hdr =
+        ObjectHeader::parse(file_data, addr as usize, sb.offset_size, sb.length_size).unwrap();
+    let msg = |t: MessageType| hdr.messages.iter().find(|m| m.msg_type == t).unwrap();
+    let ds = Dataspace::parse(&msg(MessageType::Dataspace).data, sb.length_size).unwrap();
+    let (dt, _) = Datatype::parse(&msg(MessageType::Datatype).data).unwrap();
+    let layout = DataLayout::parse(
+        &msg(MessageType::DataLayout).data,
+        sb.offset_size,
+        sb.length_size,
+    )
+    .unwrap();
+    let fill = clawhdf5_format::fill_value::dataset_fill_value_in(
+        file_data,
+        &hdr.messages,
+        sb.offset_size,
+        sb.length_size,
+    )
+    .unwrap();
+    let v = clawhdf5_format::vds::read_virtual_dataset(
+        file_data,
+        &layout,
+        &ds,
+        &dt,
+        fill.as_deref(),
+        sb.offset_size,
+        sb.length_size,
+        None,
+    )
+    .unwrap();
+    assert_eq!(v.dims, ds.dimensions);
+    (v.data, dt)
+}
+
 /// Helper: read any dataset (contiguous or chunked) as f64.
 fn read_dataset_f64_any(bytes: &[u8], path: &str) -> Vec<f64> {
     let sig = find_signature(bytes).unwrap();
@@ -672,7 +711,7 @@ fn v4_virtual_dataset_same_file_read() {
     //   virt[4:8]  <- (unmapped)                              => fill 0
     //   virt[8:12] <- src_b[0:4]  (ALL)                       => 20,21,22,23
     let file_data = include_bytes!("fixtures/vds_same_file.h5");
-    let (raw, datatype, _) = read_chunked_dataset(file_data, "virt");
+    let (raw, datatype) = read_virtual_fixture(file_data, "virt");
     let values = read_as_i32(&raw, &datatype).unwrap();
     assert_eq!(
         values,
@@ -681,6 +720,38 @@ fn v4_virtual_dataset_same_file_read() {
     );
 }
 
+#[test]
+fn v4_virtual_dataset_raw_api_refuses_to_guess_the_fill_value() {
+    // The raw read API has no fill value message, so a virtual dataset with an
+    // unmapped region is an error there instead of zeros that may be wrong.
+    let file_data = include_bytes!("fixtures/vds_same_file.h5");
+    let sig = find_signature(file_data).unwrap();
+    let sb = Superblock::parse(file_data, sig).unwrap();
+    let addr = resolve_path_any(file_data, &sb, "virt").unwrap();
+    let hdr =
+        ObjectHeader::parse(file_data, addr as usize, sb.offset_size, sb.length_size).unwrap();
+    let msg = |t: MessageType| hdr.messages.iter().find(|m| m.msg_type == t).unwrap();
+    let ds = Dataspace::parse(&msg(MessageType::Dataspace).data, sb.length_size).unwrap();
+    let (dt, _) = Datatype::parse(&msg(MessageType::Datatype).data).unwrap();
+    let layout = DataLayout::parse(
+        &msg(MessageType::DataLayout).data,
+        sb.offset_size,
+        sb.length_size,
+    )
+    .unwrap();
+    let err = read_raw_data_full(
+        file_data,
+        &layout,
+        &ds,
+        &dt,
+        None,
+        sb.offset_size,
+        sb.length_size,
+    )
+    .unwrap_err();
+    assert!(err.to_string().contains("fill value"), "{err}");
+}
+
 #[test]
 fn v4_virtual_dataset_2d_same_file_read() {
     // A 4x4 virtual dataset assembled from two 2x2 same-file sources placed as
@@ -689,7 +760,7 @@ fn v4_virtual_dataset_2d_same_file_read() {
     //   virt[2:4,2:4] <- src_b = [[5,6],[7,8]]
     //   everything else -> fill 0
     let file_data = include_bytes!("fixtures/vds_2d_same_file.h5");
-    let (raw, datatype, _) = read_chunked_dataset(file_data, "virt");
+    let (raw, datatype) = read_virtual_fixture(file_data, "virt");
     let values = read_as_i32(&raw, &datatype).unwrap();
     assert_eq!(
         values,

crates/clawhdf5-io/src/async_read.rs

@@ -268,28 +268,31 @@ impl AsyncHDF5File {
     ///
     /// Reads the entire file into memory, then parses the superblock.
     pub async fn open<R: AsyncHDF5Read>(reader: &R) -> Result<Self, AsyncHDF5Error> {
-        let data = reader.read_all().await?;
+        Self::from_bytes(reader.read_all().await?)
-        let sig_offset = find_signature(&data)?;
-        let superblock = Superblock::parse(&data, sig_offset)?;
-        Ok(Self { data, superblock })
     }
 
     /// Open an HDF5 file asynchronously from a file path.
     pub async fn open_path<P: AsRef<Path>>(path: P) -> Result<Self, AsyncHDF5Error> {
-        let data = tokio::fs::read(path).await?;
+        Self::from_bytes(tokio::fs::read(path).await?)
-        let sig_offset = find_signature(&data)?;
-        let superblock = Superblock::parse(&data, sig_offset)?;
-        Ok(Self { data, superblock })
     }
 
     /// Open an HDF5 file from bytes already in memory.
-    pub fn from_bytes(data: Vec<u8>) -> Result<Self, AsyncHDF5Error> {
+    pub fn from_bytes(mut data: Vec<u8>) -> Result<Self, AsyncHDF5Error> {
-        let sig_offset = find_signature(&data)?;
+        // HDF5 addresses are relative to the superblock: drop any user block
-        let superblock = Superblock::parse(&data, sig_offset)?;
+        // so they index `data` directly.
+        let user_block = find_signature(&data)?;
+        let whole_len = data.len() as u64;
+        data.drain(..user_block);
+        let superblock = Superblock::parse(&data, 0)?;
+        // Refuse a truncated file, and keep nothing past the end of file the
+        // superblock records, as libhdf5 does.
+        let end = superblock.data_end(user_block as u64, whole_len)?;
+        data.truncate(end as usize);
         Ok(Self { data, superblock })
     }
 
-    /// Access the raw file bytes.
+    /// Access the file bytes from the superblock on (any user block is
+    /// dropped on open).
     pub fn as_bytes(&self) -> &[u8] {
         &self.data
     }
@@ -314,7 +317,7 @@ impl AsyncHDF5File {
 
         let dt_msg =
             find_msg(&header, MessageType::Datatype).ok_or(FormatError::DatasetMissingData)?;
-        let (datatype, _) = Datatype::parse(&dt_msg.data)?;
+        let (datatype, _) = Datatype::parse_in_header(&dt_msg.data, header.version)?;
 
         let ds_msg =
             find_msg(&header, MessageType::Dataspace).ok_or(FormatError::DatasetMissingShape)?;
@@ -607,6 +610,28 @@ mod tests {
         tokio::fs::remove_file(&path).await.ok();
     }
 
+    /// As libhdf5 does: a truncated file is refused, and bytes past the end
+    /// of file the superblock records are dropped.
+    #[tokio::test]
+    async fn async_refuses_truncated_files_and_drops_trailing_bytes() {
+        let bytes = make_test_hdf5_f64("v", &[1.0, 2.0]);
+        let truncated = bytes[..bytes.len() - 8].to_vec();
+        let err = AsyncHDF5File::from_bytes(truncated).err().unwrap();
+        assert!(
+            matches!(
+                err,
+                AsyncHDF5Error::Format(FormatError::TruncatedFile { .. })
+            ),
+            "{err}"
+        );
+
+        let mut appended = bytes.clone();
+        appended.extend_from_slice(&[0xAB; 64]);
+        let file = AsyncHDF5File::from_bytes(appended).unwrap();
+        assert_eq!(file.as_bytes().len(), bytes.len());
+        assert_eq!(file.read_f64("v").await.unwrap(), vec![1.0, 2.0]);
+    }
+
     #[tokio::test]
     async fn async_error_display() {
         let io_err = AsyncHDF5Error::Io(io::Error::new(io::ErrorKind::NotFound, "gone"));

crates/clawhdf5-io/src/mpi_vol.rs

@@ -180,8 +180,7 @@ fn mpi_collective_read(vol: &MpiVol, location: &str, path: &str) -> Result<Vec<u
     use clawhdf5_format::{
         data_layout::DataLayout, data_read::read_raw_data_full, dataspace::Dataspace,
         datatype::Datatype, filter_pipeline::FilterPipeline, group_v2::resolve_path_any,
-        message_type::MessageType, object_header::ObjectHeader, signature::find_signature,
+        message_type::MessageType, object_header::ObjectHeader,
-        superblock::Superblock,
     };
     use mpi::traits::*;
 
@@ -192,9 +191,10 @@ fn mpi_collective_read(vol: &MpiVol, location: &str, path: &str) -> Result<Vec<u
     let mut len_buf = [0usize; 1];
 
     if rank == 0 {
-        let bytes = std::fs::read(location).map_err(VolError::Io)?;
+        let file = std::fs::read(location).map_err(VolError::Io)?;
-        let sig = find_signature(&bytes).map_err(|e| VolError::DataError(e.to_string()))?;
+        // From the superblock to the recorded end of file; truncated files
-        let sb = Superblock::parse(&bytes, sig).map_err(|e| VolError::DataError(e.to_string()))?;
+        // are refused.
+        let (bytes, sb) = crate::vol::hdf5_view(&file)?;
         let addr = resolve_path_any(&bytes, &sb, path)
             .map_err(|e| VolError::NotFound(format!("{path}: {e}")))?;
         let oh = ObjectHeader::parse(&bytes, addr as usize, sb.offset_size, sb.length_size)
@@ -204,8 +204,8 @@ fn mpi_collective_read(vol: &MpiVol, location: &str, path: &str) -> Result<Vec<u
             .iter()
             .find(|m| m.msg_type == MessageType::Datatype)
             .ok_or_else(|| VolError::DataError("no datatype".into()))?;
-        let (datatype, _) =
+        let (datatype, _) = Datatype::parse_in_header(&dt.data, oh.version)
-            Datatype::parse(&dt.data).map_err(|e| VolError::DataError(e.to_string()))?;
+            .map_err(|e| VolError::DataError(e.to_string()))?;
         let ds = oh
             .messages
             .iter()

crates/clawhdf5-io/src/vol.rs

@@ -210,6 +210,24 @@ pub struct NativeVol {
     location: Option<String>,
 }
 
+/// The HDF5 bytes of a whole file and its superblock: from the superblock
+/// (addresses are relative to it, so any user block is skipped) to the end
+/// of file the superblock records. A file shorter than that is truncated
+/// and refused, and nothing past it is read, as in libhdf5.
+pub(crate) fn hdf5_view(
+    whole: &[u8],
+) -> Result<(&[u8], clawhdf5_format::superblock::Superblock), VolError> {
+    use clawhdf5_format::{signature::split_user_block, superblock::Superblock};
+    let err = |e: clawhdf5_format::error::FormatError| VolError::DataError(e.to_string());
+    let (user_block, data) = split_user_block(whole).map_err(err)?;
+    let sb = Superblock::parse(data, 0).map_err(err)?;
+    let end = sb
+        .data_end(user_block.len() as u64, whole.len() as u64)
+        .map_err(err)?;
+    // data_end is at most the file length less the user block.
+    Ok((&data[..end as usize], sb))
+}
+
 impl NativeVol {
     /// Create a new native VOL connector.
     pub fn new() -> Self {
@@ -264,6 +282,8 @@ impl VirtualObjectLayer for NativeVol {
 
     fn open(&mut self, location: &str) -> Result<(), VolError> {
         let data = std::fs::read(location)?;
+        // Refuse a truncated file at open, as libhdf5 does.
+        hdf5_view(&data)?;
         self.data = Some(data);
         self.location = Some(location.to_string());
         Ok(())
@@ -283,12 +303,10 @@ impl VirtualObjectLayer for NativeVol {
         use clawhdf5_format::{
             data_layout::DataLayout, data_read::read_raw_data_full, dataspace::Dataspace,
             datatype::Datatype, filter_pipeline::FilterPipeline, group_v2::resolve_path_any,
-            message_type::MessageType, object_header::ObjectHeader, signature::find_signature,
+            message_type::MessageType, object_header::ObjectHeader,
-            superblock::Superblock,
         };
 
-        let sig = find_signature(data).map_err(|e| VolError::DataError(e.to_string()))?;
+        let (data, sb) = hdf5_view(data)?;
-        let sb = Superblock::parse(data, sig).map_err(|e| VolError::DataError(e.to_string()))?;
         let addr = resolve_path_any(data, &sb, path)
             .map_err(|e| VolError::NotFound(format!("{path}: {e}")))?;
 
@@ -300,8 +318,8 @@ impl VirtualObjectLayer for NativeVol {
             .iter()
             .find(|m| m.msg_type == MessageType::Datatype)
             .ok_or_else(|| VolError::DataError("missing datatype".into()))?;
-        let (datatype, _) =
+        let (datatype, _) = Datatype::parse_in_header(&dt_msg.data, header.version)
-            Datatype::parse(&dt_msg.data).map_err(|e| VolError::DataError(e.to_string()))?;
+            .map_err(|e| VolError::DataError(e.to_string()))?;
 
         let ds_msg = header
             .messages
@@ -386,6 +404,36 @@ mod tests {
         assert!(vol.as_bytes().is_none());
     }
 
+    /// As libhdf5 does: a file shorter than the end of file its superblock
+    /// records is truncated and refused (at open, and when read from
+    /// memory), and bytes appended past that end are not part of the file.
+    #[test]
+    fn native_vol_refuses_truncated_files_and_ignores_trailing_bytes() {
+        use clawhdf5_format::file_writer::FileWriter as FmtWriter;
+
+        let mut fw = FmtWriter::new();
+        fw.create_dataset("x").with_f64_data(&[1.0, 2.0, 3.0]);
+        let bytes = fw.finish().unwrap();
+
+        let truncated = bytes[..bytes.len() - 8].to_vec();
+        let err = NativeVol::from_bytes(truncated.clone())
+            .read_dataset("x")
+            .unwrap_err();
+        assert!(err.to_string().contains("truncated"), "{err}");
+        let dir = std::env::temp_dir().join(format!("clawhdf5_vol_trunc_{}", std::process::id()));
+        std::fs::create_dir_all(&dir).unwrap();
+        let path = dir.join("truncated.h5");
+        std::fs::write(&path, &truncated).unwrap();
+        let err = NativeVol::open_path(path.to_str().unwrap()).err().unwrap();
+        assert!(err.to_string().contains("truncated"), "{err}");
+        std::fs::remove_dir_all(&dir).ok();
+
+        let mut appended = bytes.clone();
+        appended.extend_from_slice(&[0xAB; 64]);
+        let raw = NativeVol::from_bytes(appended).read_dataset("x").unwrap();
+        assert_eq!(raw.len(), 24);
+    }
+
     #[test]
     fn vol_error_display() {
         let err = VolError::Unsupported("read_dataset".into());

crates/clawhdf5-tools/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "clawhdf5-tools"
+version = "2.7.0"
+edition = "2024"
+rust-version.workspace = true
+license = "MIT"
+description = "h5rs: pure-Rust HDF5 command-line tools (ls, dump, stat, diff, check) without libhdf5"
+repository = "https://git.redclaw.dev/quantumclaw/clawhdf5"
+keywords = ["hdf5", "h5dump", "h5ls", "cli", "science"]
+categories = ["command-line-utilities", "science"]
+readme = "README.md"
+
+[[bin]]
+name = "h5rs"
+path = "src/main.rs"
+
+[dependencies]
+clawhdf5 = { path = "../clawhdf5", version = "2.7.0" }
+clawhdf5-format = { path = "../clawhdf5-format", version = "2.7.0" }
+serde_json = "1"
+
+[dev-dependencies]
+tempfile = { workspace = true }

crates/clawhdf5-tools/README.md

@@ -0,0 +1,278 @@
+# clawhdf5-tools: `h5rs`
+
+HDF5 command-line tools in pure Rust, built only on the `clawhdf5` facade and
+`clawhdf5-format`. No libhdf5 and no C code, so the binary also builds as a
+fully static executable: `cargo build --release -p clawhdf5-tools --target
+x86_64-unknown-linux-musl` gives a static-pie `h5rs` of 1.8 MB with no
+shared-library dependencies (built and run on tank, 2026-09-26, Rust 1.98).
+
+| Command | Modelled on | What it does |
+|---------|-------------|--------------|
+| `h5rs ls` | `h5ls` | list objects: name, kind, shape, datatype; `-v` adds layout, chunking, storage, filters, attributes |
+| `h5rs dump` | `h5dump` | the file's structure and values as DDL text, or as JSON (hdf5-json layout) |
+| `h5rs stat` | `h5stat` | object, link, rank, layout, filter and attribute counts; raw-data and file size |
+| `h5rs diff` | `h5diff` | structural and value differences between two files or objects |
+| `h5rs check` | `h5check` | structural validator: every object and header message, the checksums of version 2+ structures, chunk index consistency |
+
+```bash
+cargo install --path crates/clawhdf5-tools     # or: cargo build --release -p clawhdf5-tools
+h5rs --help
+h5rs <command> --help
+```
+
+Every command takes `--max-bytes N` where it reads values (default 1 GiB): a
+dataset whose dataspace claims more than that is reported instead of read, so
+a corrupt size cannot exhaust memory.
+
+## `h5rs ls`
+
+```console
+$ h5rs ls -r data.h5            # an extract of the output
+/                        Group
+/external                External Link {other.h5//x}
+/grp                     Group
+/grp/ext2                Dataset {6/Inf, 10/Inf} float32
+/grp/gz                  Dataset {1000} int32
+/hard2                   Group, same as /grp/sub
+/named_t                 Type
+/soft                    Soft Link {/contig}
+```
+
+The first two columns are h5ls's (`h5ls -r` prints the same text); h5rs adds
+the datatype. `FILE/path` lists a group's members or one dataset, as h5ls
+does. `-v` prints, per object:
+
+```console
+$ h5rs ls -v data.h5/grp/gz
+gz                       Dataset {1000/1000}
+    Address:   1120
+    Links:     1
+    Layout:    chunked (fixed array index)
+    Chunks:    {100} 400 bytes
+    Storage:   4000 logical bytes, 1237 allocated bytes, 323.36% utilization
+    Filter-0:  shuffle-2 OPT {4}
+    Filter-1:  deflate-1 OPT {4}
+    Filter-2:  fletcher32-3
+    Type:      32-bit little-endian integer
+```
+
+## `h5rs dump`
+
+```console
+$ h5rs dump data.h5            # DDL, like h5dump
+$ h5rs dump -A data.h5         # no dataset values (attributes still shown), like h5dump -A
+$ h5rs dump -d /grp/gz data.h5 # one dataset
+$ h5rs dump -p data.h5         # also STORAGE_LAYOUT and FILTERS blocks
+$ h5rs dump --json data.h5     # hdf5-json
+```
+
+The DDL output is h5dump's: on the test files of `tests/gen_files.py`
+(compact, contiguous and chunked datasets with every chunk index; v1 and v2
+groups; integers of both byte orders, floats, compound with an array member,
+enum, fixed and variable-length strings; soft, external and hard links; a
+named datatype; compact and dense attributes) `h5rs dump` and `h5rs dump -A`
+print the same bytes as h5dump 1.14.6 and as Debian's h5dump 1.14.5 (the
+`hdf5-tools` package CI installs in `rust:latest`; the whole interop suite
+was run in that image on 2026-09-26) — `dump_matches_h5dump` in
+`tests/h5rs_interop.rs` checks this, and `dump_shows_nul_padding_in_nested_strings`
+that null-padded strings show their NULs (`"a\000b"`) at any depth, as
+h5dump's do. Not covered by those tests: references, opaque, bitfield,
+variable-length sequences and virtual datasets. Known differences from
+h5dump:
+
+- Floats print at their own precision (a `float32` 0.1 prints as `0.1`),
+  which for some values is more digits than h5dump's `%g`.
+- A compound nested in a compound prints inline (`{ 1, 2.5 }`) where
+  h5dump prints it as an indented block, one member per line; only the
+  outer compound is a block.
+- `long double` (x87 80-bit) and other floats wider than 64 bits: the
+  datatype is printed as an `H5T_FLOAT { ... }` block instead of h5dump's
+  one-line description, and each value as `<error: ...>`; `dump` then
+  exits 1. The library cannot convert them (see `docs/known-issues.md`).
+- The `-p` block is h5rs's own (it names the chunk index), not h5dump's.
+
+### JSON schema
+
+`--json` follows the HDF Group's [hdf5-json](https://github.com/HDFGroup/hdf5-json)
+layout:
+
+```json
+{
+  "apiVersion": "1.1.1",
+  "root": "g-0000000000000060",
+  "groups":    { "<id>": { "alias": ["/grp"], "attributes": [...], "links": [...] } },
+  "datasets":  { "<id>": { "alias": [...], "attributes": [...], "shape": {...},
+                           "type": {...}, "creationProperties": {...}, "value": ... } },
+  "datatypes": { "<id>": { "alias": [...], "attributes": [...], "type": {...} } }
+}
+```
+
+- **ids** are `g-`/`d-`/`t-` plus the object header address in 16 hex digits
+  (hdf5-json uses UUIDs; these are stable for a given file). `alias` lists
+  every path that reaches the object.
+- **links**: `{"class": "H5L_TYPE_HARD", "title", "collection", "id"}`,
+  `{"class": "H5L_TYPE_SOFT", "title", "h5path"}`,
+  `{"class": "H5L_TYPE_EXTERNAL", "title", "file", "h5path"}`,
+  `{"class": "H5L_TYPE_USER_DEFINED", "title", "linkClass"}`.
+- **shape**: `{"class": "H5S_NULL"}`, `{"class": "H5S_SCALAR"}` or
+  `{"class": "H5S_SIMPLE", "dims": [...], "maxdims": [...]}` with
+  `"H5S_UNLIMITED"` for an unlimited dimension.
+- **type**: `{"class": "H5T_INTEGER" | "H5T_FLOAT" | "H5T_BITFIELD", "base":
+  "H5T_STD_I32LE" ...}`, `{"class": "H5T_STRING", "charSet", "strPad",
+  "length": n | "H5T_VARIABLE"}`, `{"class": "H5T_COMPOUND", "fields": [{"name",
+  "type"}]}`, `{"class": "H5T_ARRAY", "base", "dims"}`, `{"class": "H5T_ENUM",
+  "base", "mapping": {"NAME": value}}`, `{"class": "H5T_VLEN", "base"}`,
+  `{"class": "H5T_OPAQUE", "size", "tag"}`, `{"class": "H5T_REFERENCE", "base":
+  "H5T_STD_REF_OBJ" | "H5T_STD_REF_DSETREG" | "H5T_STD_REF"}`.
+- **value**: nested lists in the dataset's shape (a scalar is the bare value,
+  a null dataspace `null`). A compound element is a list of its members, an
+  enum element its integer value, a string a JSON string, opaque/bitfield data
+  a `0x...` hex string, an object reference the referenced object's path,
+  NaN/infinities the strings `"NaN"`, `"Infinity"`, `"-Infinity"`, and an
+  integer beyond 64 bits a decimal string.
+- **creationProperties**: `layout` (`{"class": "H5D_CHUNKED", "dims": [...]}`
+  etc.) and `filters` (`[{"id", "name", "class", "parameters"}]`).
+
+A value that cannot be read is replaced by `"value_error": "<reason>"` and
+the command exits 1.
+
+## `h5rs stat`
+
+Prints h5stat's report sections with the same labels for the facts it
+computes — object and link counts, max links to an object, max objects in a
+group, dataset ranks, layout counts, filter counts, attribute counts, total
+raw data size and total file size (all equal to h5stat's on the test files;
+`stat_matches_h5stat` checks them). It does not break metadata space down by
+structure as h5stat does; it reports metadata and free space as one figure.
+
+## `h5rs diff`
+
+```console
+$ h5rs diff a.h5 b.h5                # whole files
+$ h5rs diff a.h5 b.h5 /grp           # one object and everything below it
+$ h5rs diff a.h5 b.h5 /x /y          # different paths in each
+$ h5rs diff -r a.h5 b.h5 /d          # list every differing element
+$ h5rs diff -d 0.001 a.h5 b.h5       # |a - b| > 0.001 is a difference
+$ h5rs diff -p 0.01 a.h5 b.h5        # |a - b| / |a| > 1% is a difference
+$ h5rs diff --follow-symlinks a.h5 b.h5 /lnk   # what the soft link /lnk leads to
+$ h5rs diff -r -n 10 a.h5 b.h5       # list at most 10 differing elements per object
+```
+
+The options are named as h5diff's: `-n N`/`--count=N` limits the listed
+elements, `-c`/`--compare` (list objects that are not comparable) is
+accepted and always in effect, and `--delta=D`, `--relative=R` work too.
+
+Integers are compared in integer arithmetic, with or without a tolerance, so
+64-bit values beyond 2^53 lose no precision (`-d 0` tells 2^60 from
+2^60 + 1). A relative tolerance below the f64 epsilon (2.2e-16) compares
+exactly, as h5diff's does.
+
+Exit status: 0 no differences, 1 differences, 2 error — the same as h5diff's
+on the cases `diff_exit_codes_match_h5diff` runs. Compared: which objects
+exist, their kinds, datatypes and shapes, attribute sets and values, dataset
+values, and soft/external link targets. A soft link — including an OBJ
+that is itself a soft link — is compared as a link, by its target path, as
+h5diff does; `--follow-symlinks` compares the objects soft links lead to
+instead (and walks into soft-linked groups), and two dangling links are then
+the same, as in h5diff. External links are always compared by target (file
+and path): `--follow-symlinks` does not open other files, where h5diff's
+does. Two dangling soft links with different targets are a difference
+(h5diff reports them with `-r`/`-v` but exits 0 without). Every path is compared: an object
+hard-linked under two names is compared under both, with everything below
+it, so a file that shares one object between two names equals a file that
+stores two identical copies. Differences from h5diff, on purpose: objects
+that cannot be compared (different shapes or datatype classes) count as a
+difference (h5diff warns and exits 0); two NaNs are equal; and the members
+of a group reached by a second hard link are compared (h5diff lists them in
+one file only and exits 1).
+
+## `h5rs check`
+
+```console
+$ h5rs check data.h5
+checked data.h5: superblock v3, 33 objects (4 groups, 28 datasets, 1 named datatypes), 139 header messages, 34 chunks
+checksums verified: superblock 1, v2 object headers 33, v2 B-trees 3, fractal heaps 3 (+3 blocks), chunk indexes 5
+no problems found
+
+$ h5rs check damaged.h5          # one bit flipped in /grp/ext1's chunk index header
+problem: 0x79c /grp/ext1: chunk index (extensible array): checksum mismatch: expected 0xcfe2391f, computed 0x2dd59bed
+checked damaged.h5: superblock v3, 33 objects (4 groups, 28 datasets, 1 named datatypes), 139 header messages, 27 chunks
+checksums verified: superblock 1, v2 object headers 33, v2 B-trees 3, fractal heaps 3 (+3 blocks), chunk indexes 4
+1 problem found
+```
+
+Each problem line is `problem: <address> <object path>: <what>`; addresses
+are HDF5 addresses (relative to the superblock, as h5dump and h5ls print
+them).
+
+It walks every object reachable from the root group (and the superblock
+extension) and checks:
+
+- the superblock (and its checksum, version 2+) and that the file is not
+  shorter than the superblock's end-of-file address;
+- every object header, with the checksum of version 2 headers and of their
+  continuation chunks, and every header message parsed by type (dataspace,
+  datatype, fill value, layout, filter pipeline, attributes, link info,
+  links, group info, symbol table), including shared messages;
+- groups: the symbol table (v1 B-tree, local heap, symbol nodes), or the
+  links, and for dense storage the fractal heap — header, and every direct
+  and indirect block with its checksum, back-pointer and heap offset — and
+  the v2 B-tree name and creation-order indexes (every node's checksum, and
+  the record count against the header's);
+- dense attribute storage the same way;
+- datasets: the layout against the dataspace and datatype (compact and
+  contiguous sizes, chunk rank), and for chunked datasets the whole chunk
+  index (v1 B-tree, single chunk, implicit, fixed array, extensible array,
+  v2 B-tree, with the checksums of the last three): every chunk's offset must
+  be a multiple of the chunk size and inside the extent, appear once, and
+  have a plausible size;
+- that all raw data (contiguous blocks and chunks) lies inside the file and
+  no two pieces overlap.
+
+`--data` also reads every dataset, decoding every chunk through its filters
+(which catches corrupt compressed data and Fletcher-32 mismatches), and
+follows every variable-length element (strings and sequences, also inside
+compounds and arrays) of every dataset and attribute into its global heap
+collection: a collection that does not parse, a missing heap object, or a
+sequence longer than its heap object is a problem at the collection's
+address. Data the
+tool cannot decode (a filter it does not implement, such as szip, or a
+dataset over `--max-bytes`) is a `note:`, not a problem. Every problem is
+printed with the address of the structure involved; the exit status is 0
+when there are none, 1 when there are, 2 for a usage error or a missing
+file.
+
+libhdf5's h5check understands only the HDF5 1.8 file format; `h5rs check`
+also covers the structures HDF5 1.10+ writes (fixed/extensible array and v2
+B-tree chunk indexes, and version 3 superblocks).
+
+What it does not check: free-space manager and shared-message (SOHM) table
+checksums, global heap collections no variable-length value points into (and
+none at all without `--data`), and objects reachable only by external links. It validates with
+clawhdf5's parsers, so it accepts what they accept: some header damage that
+libhdf5 refuses goes unreported. Of the 150 CVE and fuzzer files of the
+HDF Group's `cve_hdf5` corpus (`cvefiles/` and `fuzzerfiles/`),
+`check --data` passes 16, and h5dump 1.14.6 rejects 9 of those (tank,
+2026-09-26, `h5rs check --data F` and `h5dump F` per file; before the
+library's header checks it passed 28, of which h5dump rejects 21).
+
+## Robustness
+
+A panic is a bug: `h5rs` catches it, prints `internal error`, and exits 3
+(`check` records it against the object and carries on). `scripts/h5rs-fuzz.sh`
+runs every subcommand over every file of a corpus (by default the HDF Group's
+CVE reproducers, fetched by `conformance/fetch-corpus.sh`), optionally with
+byte-flipped copies (`MUTATE=N`), under a timeout and a memory limit, with
+overflow checks on, and fails on any panic, crash or hang.
+`scripts/h5rs-check-ok-files.sh` runs `check --data` over the conformance
+files that both clawhdf5 and h5py read in full, which must all pass.
+
+## Tests
+
+```bash
+CLAWHDF5_PYTHON=.venv/bin/python CLAWHDF5_REQUIRE_INTEROP=1 cargo test -p clawhdf5-tools
+```
+
+The interop tests write their files with h5py and compare with h5ls, h5stat,
+h5dump and h5diff; each skips when what it needs is missing unless
+`CLAWHDF5_REQUIRE_INTEROP=1`.

crates/clawhdf5-tools/src/cli.rs

@@ -0,0 +1,52 @@
+//! Argument handling and output plumbing shared by the subcommands.
+
+use std::io::Write;
+
+/// Where a subcommand writes: `o` for results, `e` for diagnostics.
+pub struct Out<'a> {
+    pub o: &'a mut dyn Write,
+    pub e: &'a mut dyn Write,
+}
+
+/// The arguments after the subcommand name.
+pub struct Args {
+    cmd: &'static str,
+    rest: std::collections::VecDeque<String>,
+}
+
+impl Args {
+    pub fn new(cmd: &'static str, rest: Vec<String>) -> Self {
+        Self {
+            cmd,
+            rest: rest.into(),
+        }
+    }
+
+    #[allow(clippy::should_implement_trait)]
+    pub fn next(&mut self) -> Option<String> {
+        self.rest.pop_front()
+    }
+
+    /// Put an option and its value back, to be read next (for the
+    /// `--name=value` form).
+    pub fn push_front(&mut self, name: String, value: String) {
+        self.rest.push_front(value);
+        self.rest.push_front(name);
+    }
+
+    /// The value after an option such as `--max-bytes`.
+    pub fn value(&mut self) -> Option<String> {
+        self.rest.pop_front()
+    }
+
+    /// A numeric option value.
+    pub fn number<T: std::str::FromStr>(&mut self) -> Option<T> {
+        self.rest.pop_front().and_then(|s| s.parse().ok())
+    }
+
+    /// Report a usage problem; exit status 2.
+    pub fn usage_error(&self, out: &mut Out, msg: &str, usage: &str) -> std::io::Result<i32> {
+        writeln!(out.e, "h5rs {}: {msg}\n\n{usage}", self.cmd)?;
+        Ok(2)
+    }
+}

crates/clawhdf5-tools/src/diff.rs

@@ -0,0 +1,904 @@
+//! `h5rs diff`: compare two files (or two objects) like h5diff.
+
+use std::cell::OnceCell;
+use std::collections::{BTreeMap, HashMap};
+use std::rc::Rc;
+
+use clawhdf5_format::attribute::AttributeMessage;
+use clawhdf5_format::dataspace::{Dataspace, DataspaceType};
+use clawhdf5_format::datatype::Datatype;
+use clawhdf5_format::object_header::ObjectHeader;
+
+use crate::cli::{Args, Out};
+use crate::h5::{H5, Kind, Link, LinkKind};
+use crate::value::{self, Decoder, Value};
+
+pub const USAGE: &str = "\
+usage: h5rs diff [options] FILE1 FILE2 [OBJ1 [OBJ2]]
+
+Compare FILE1 and FILE2 (or OBJ1 in FILE1 with OBJ2 in FILE2, and everything
+below them): the objects present, their kinds, datatypes, shapes, attribute
+sets and values, and soft/external link targets (an OBJ that is itself a
+soft link is compared as a link, as h5diff does, unless --follow-symlinks).
+
+  -r, --report       list every differing element (position, values, difference)
+  -q, --quiet        print nothing; only the exit status
+  --follow-symlinks  compare the objects soft links lead to (and walk into
+                     soft-linked groups) instead of the links' target paths;
+                     two dangling links are the same
+  -d, --delta D      numbers differ only when |a - b| > D
+  -p, --relative R   numbers differ only when |a - b| / |a| > R (an R below
+                     the f64 epsilon, 2.2e-16, compares exactly, as h5diff)
+  -n, --count N      list at most N differing elements per object with -r
+  -c, --compare      list objects that are not comparable (always done;
+                     accepted for h5diff compatibility)
+  --max-bytes N      largest dataset read (default 1 GiB); a larger one is an error
+
+Two NaNs compare equal. Unlike h5diff, objects that cannot be compared
+(different kinds, datatype classes or shapes) count as a difference.
+
+Exit status: 0 no differences, 1 differences found, 2 error (a file or
+object could not be opened or read).";
+
+#[derive(Clone, Copy)]
+enum Tol {
+    Exact,
+    Delta(f64),
+    Relative(f64),
+}
+
+struct Opts {
+    report: bool,
+    quiet: bool,
+    tol: Tol,
+    count: usize,
+    /// Compare the objects soft links lead to, not the links' targets.
+    follow: bool,
+}
+
+/// What a relative path names in one file.
+#[derive(Clone)]
+enum Entry {
+    Obj(u64, Kind),
+    Soft(String),
+    External(String, String),
+    UserDefined(u8),
+    Broken(String),
+}
+
+struct Side<'a> {
+    h5: &'a H5,
+    label: String,
+    base: String,
+    /// Relative path -> what it is.
+    entries: BTreeMap<String, Entry>,
+    /// Address -> relative path, for comparing references.
+    rel_of: OnceCell<HashMap<u64, String>>,
+}
+
+impl Side<'_> {
+    fn full(&self, rel: &str) -> String {
+        if rel.is_empty() {
+            if self.base.is_empty() {
+                "/".into()
+            } else {
+                self.base.clone()
+            }
+        } else {
+            format!("{}{rel}", self.base)
+        }
+    }
+
+    fn rel_paths(&self) -> &HashMap<u64, String> {
+        self.rel_of.get_or_init(|| {
+            let mut m = HashMap::new();
+            let _ = self.h5.walk(|it| {
+                if let (Some(a), None) = (it.addr, it.first_path) {
+                    m.entry(a).or_insert_with(|| it.path.to_string());
+                }
+            });
+            m
+        })
+    }
+}
+
+struct Diff {
+    opts: Opts,
+    diffs: u64,
+    /// Differences already reported under an object heading.
+    per_object: u64,
+    errors: u64,
+}
+
+pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result<i32> {
+    let mut opts = Opts {
+        report: false,
+        quiet: false,
+        tol: Tol::Exact,
+        count: usize::MAX,
+        follow: false,
+    };
+    let mut max_bytes = None;
+    let mut pos = Vec::new();
+    while let Some(a) = args.next() {
+        match a.as_str() {
+            "-r" | "--report" => opts.report = true,
+            "-q" | "--quiet" => opts.quiet = true,
+            "--follow-symlinks" => opts.follow = true,
+            "-d" | "--delta" => match args.number::<f64>() {
+                Some(d) if d >= 0.0 => opts.tol = Tol::Delta(d),
+                _ => return args.usage_error(out, "--delta needs a number >= 0", USAGE),
+            },
+            "-p" | "--relative" => match args.number::<f64>() {
+                // As h5diff: a relative tolerance below the f64 epsilon
+                // cannot be told from rounding, so it compares exactly.
+                Some(r) if r >= 0.0 => {
+                    opts.tol = if r < f64::EPSILON {
+                        Tol::Exact
+                    } else {
+                        Tol::Relative(r)
+                    }
+                }
+                _ => return args.usage_error(out, "--relative needs a number >= 0", USAGE),
+            },
+            // h5diff's -c: h5rs always lists them.
+            "-c" | "--compare" => {}
+            "-n" | "--count" => match args.number() {
+                Some(n) => opts.count = n,
+                None => return args.usage_error(out, "--count needs a number", USAGE),
+            },
+            "--max-bytes" => match args.number() {
+                Some(n) => max_bytes = Some(n),
+                None => return args.usage_error(out, "--max-bytes needs a number", USAGE),
+            },
+            "-h" | "--help" => {
+                writeln!(out.o, "{USAGE}")?;
+                return Ok(0);
+            }
+            // h5diff's --count=N, --delta=D, --relative=R forms.
+            s if s.starts_with("--") && s.contains('=') => {
+                let (k, v) = s.split_once('=').unwrap_or((s, ""));
+                if !matches!(k, "--count" | "--delta" | "--relative" | "--max-bytes") {
+                    return args.usage_error(out, &format!("unknown option {a}"), USAGE);
+                }
+                args.push_front(k.to_string(), v.to_string());
+            }
+            s if s.starts_with('-') && s.len() > 1 => {
+                return args.usage_error(out, &format!("unknown option {a}"), USAGE);
+            }
+            _ => pos.push(a),
+        }
+    }
+    if pos.len() < 2 || pos.len() > 4 {
+        return args.usage_error(out, "expected FILE1 FILE2 [OBJ1 [OBJ2]]", USAGE);
+    }
+    let mut files = Vec::new();
+    for f in &pos[..2] {
+        match H5::open(std::path::Path::new(f)) {
+            Ok(mut h) => {
+                if let Some(m) = max_bytes {
+                    h.max_bytes = m;
+                }
+                files.push(h);
+            }
+            Err(e) => {
+                writeln!(out.e, "h5rs diff: {e}")?;
+                return Ok(2);
+            }
+        }
+    }
+    let obj1 = pos.get(2).cloned().unwrap_or_else(|| "/".into());
+    let obj2 = pos.get(3).cloned().unwrap_or_else(|| obj1.clone());
+    let mut sides = Vec::new();
+    for (h5, (obj, f)) in files.iter().zip([(&obj1, &pos[0]), (&obj2, &pos[1])]) {
+        let start = match start(h5, obj, opts.follow) {
+            Ok(x) => x,
+            Err(_) => {
+                writeln!(
+                    out.e,
+                    "h5rs diff: object <{obj}> could not be found in <{f}>"
+                )?;
+                return Ok(2);
+            }
+        };
+        let base = obj.trim_end_matches('/').to_string();
+        let base = if base.is_empty() || base.starts_with('/') {
+            base
+        } else {
+            format!("/{base}")
+        };
+        let collected = match start {
+            Start::Obj(addr) => collect(h5, addr, &base, opts.follow),
+            Start::Link(e) => Ok(BTreeMap::from([(String::new(), e)])),
+        };
+        let entries = match collected {
+            Ok(e) => e,
+            Err(e) => {
+                writeln!(out.e, "h5rs diff: {f}: {e}")?;
+                return Ok(2);
+            }
+        };
+        sides.push(Side {
+            h5,
+            label: f.clone(),
+            base,
+            entries,
+            rel_of: OnceCell::new(),
+        });
+    }
+    let (a, b) = (&sides[0], &sides[1]);
+    let mut d = Diff {
+        opts,
+        diffs: 0,
+        per_object: 0,
+        errors: 0,
+    };
+    let mut names: Vec<&String> = a.entries.keys().chain(b.entries.keys()).collect();
+    names.sort();
+    names.dedup();
+    for rel in names {
+        match (a.entries.get(rel), b.entries.get(rel)) {
+            (Some(_), None) => {
+                d.diffs += 1;
+                d.say(
+                    out,
+                    &format!("<{}> exists only in <{}>", a.full(rel), a.label),
+                )?;
+            }
+            (None, Some(_)) => {
+                d.diffs += 1;
+                d.say(
+                    out,
+                    &format!("<{}> exists only in <{}>", b.full(rel), b.label),
+                )?;
+            }
+            (Some(ea), Some(eb)) => d.entry(out, a, b, rel, ea, eb)?,
+            (None, None) => {}
+        }
+    }
+    // Per-object counts were printed with each object; add a total when
+    // they do not already tell the whole story.
+    if !d.opts.quiet && d.diffs > 0 && d.diffs != d.per_object {
+        writeln!(out.o, "{} difference(s) found in total", d.diffs)?;
+    }
+    Ok(if d.errors > 0 {
+        2
+    } else if d.diffs > 0 {
+        1
+    } else {
+        0
+    })
+}
+
+/// What OBJ1/OBJ2 names.
+enum Start {
+    Obj(u64),
+    /// A soft (not followed, or dangling), external or user-defined link.
+    Link(Entry),
+}
+
+/// Resolve OBJ1/OBJ2. Soft links on the way to it are followed, as in any
+/// HDF5 path; a soft link that *is* the named object is compared as a link
+/// (its target path), as h5diff does, unless `follow`.
+fn start(h5: &H5, obj: &str, follow: bool) -> crate::h5::Result<Start> {
+    let p = obj.trim_matches('/');
+    if !p.is_empty() {
+        let (parent, name) = p.rsplit_once('/').unwrap_or(("", p));
+        let link = h5
+            .resolve(parent)
+            .and_then(|a| h5.header(a))
+            .and_then(|h| h5.links(&h))
+            .ok()
+            .and_then(|ls| ls.into_iter().find(|l| l.name == name));
+        if let Some(l) = link {
+            return Ok(match l.kind {
+                LinkKind::Hard(a) => Start::Obj(a),
+                LinkKind::Soft(t) if follow => match soft_target(h5, &format!("/{parent}"), &t) {
+                    Ok(a) => Start::Obj(a),
+                    Err(_) => Start::Link(Entry::Soft(t)),
+                },
+                LinkKind::Soft(t) => Start::Link(Entry::Soft(t)),
+                LinkKind::External { file, path } => Start::Link(Entry::External(file, path)),
+                LinkKind::UserDefined(t) => Start::Link(Entry::UserDefined(t)),
+            });
+        }
+    }
+    h5.resolve(obj).map(Start::Obj)
+}
+
+/// The object a soft link in group `parent` (an absolute path) leads to.
+fn soft_target(h5: &H5, parent: &str, target: &str) -> crate::h5::Result<u64> {
+    if target.starts_with('/') {
+        h5.resolve(target)
+    } else {
+        h5.resolve(&format!("{}/{target}", parent.trim_end_matches('/')))
+    }
+}
+
+/// One object as the path walk sees it: what its paths compare as, and
+/// (for a group) its links.
+struct Node {
+    entry: Entry,
+    links: Vec<Link>,
+}
+
+/// Every path below the object at `start`, relative to it (`""` is `start`
+/// itself). Each hard link is its own path, so an object linked under two
+/// names is compared under both, with everything below it: two files that
+/// hold the same values are equal whether one shares an object between
+/// names and the other stores copies. A hard link back to an ancestor (a
+/// cycle) is recorded, not descended into. With `follow`, a soft link is
+/// walked like a hard link to its target (a dangling one stays a link);
+/// `base` is the absolute path of `start`, for relative link targets.
+fn collect(
+    h5: &H5,
+    start: u64,
+    base: &str,
+    follow: bool,
+) -> crate::h5::Result<BTreeMap<String, Entry>> {
+    let mut cache: HashMap<u64, Rc<Node>> = HashMap::new();
+    let mut node = |addr: u64| -> Rc<Node> {
+        cache
+            .entry(addr)
+            .or_insert_with(|| {
+                Rc::new(match h5.header(addr) {
+                    Err(e) => Node {
+                        entry: Entry::Broken(e.to_string()),
+                        links: Vec::new(),
+                    },
+                    Ok(h) => match Kind::of(&h) {
+                        Kind::Group => match h5.links(&h) {
+                            Ok(links) => Node {
+                                entry: Entry::Obj(addr, Kind::Group),
+                                links,
+                            },
+                            Err(e) => Node {
+                                entry: Entry::Broken(format!("links: {e}")),
+                                links: Vec::new(),
+                            },
+                        },
+                        k => Node {
+                            entry: Entry::Obj(addr, k),
+                            links: Vec::new(),
+                        },
+                    },
+                })
+            })
+            .clone()
+    };
+    let mut entries = BTreeMap::new();
+    // (address, relative path, addresses of the groups above it)
+    let mut stack: Vec<(u64, String, Rc<Vec<u64>>)> =
+        vec![(start, String::new(), Rc::new(Vec::new()))];
+    while let Some((addr, path, above)) = stack.pop() {
+        if entries.len() >= crate::h5::MAX_OBJECTS {
+            return Err(crate::h5::Error::new(format!(
+                "more than {} paths; stopped walking",
+                crate::h5::MAX_OBJECTS
+            )));
+        }
+        let n = node(addr);
+        entries.insert(path.clone(), n.entry.clone());
+        if n.links.is_empty() || above.contains(&addr) {
+            continue;
+        }
+        let mut chain = (*above).clone();
+        chain.push(addr);
+        let chain = Rc::new(chain);
+        for l in &n.links {
+            let child = format!("{path}/{}", l.name);
+            match &l.kind {
+                LinkKind::Hard(a) => stack.push((*a, child, chain.clone())),
+                LinkKind::Soft(t) if follow => match soft_target(h5, &format!("{base}{path}"), t) {
+                    Ok(a) => stack.push((a, child, chain.clone())),
+                    Err(_) => {
+                        entries.insert(child, Entry::Soft(t.clone()));
+                    }
+                },
+                LinkKind::Soft(t) => {
+                    entries.insert(child, Entry::Soft(t.clone()));
+                }
+                LinkKind::External { file, path } => {
+                    entries.insert(child, Entry::External(file.clone(), path.clone()));
+                }
+                LinkKind::UserDefined(t) => {
+                    entries.insert(child, Entry::UserDefined(*t));
+                }
+            }
+        }
+    }
+    Ok(entries)
+}
+
+fn kind_word(k: Kind) -> &'static str {
+    match k {
+        Kind::Group => "group",
+        Kind::Dataset => "dataset",
+        Kind::Datatype => "datatype",
+        Kind::Unknown => "object",
+    }
+}
+
+impl Diff {
+    fn say(&self, out: &mut Out, msg: &str) -> std::io::Result<()> {
+        if self.opts.quiet {
+            Ok(())
+        } else {
+            writeln!(out.o, "{msg}")
+        }
+    }
+
+    fn error(&mut self, out: &mut Out, msg: &str) -> std::io::Result<()> {
+        self.errors += 1;
+        writeln!(out.e, "h5rs diff: {msg}")
+    }
+
+    fn entry(
+        &mut self,
+        out: &mut Out,
+        a: &Side,
+        b: &Side,
+        rel: &str,
+        ea: &Entry,
+        eb: &Entry,
+    ) -> std::io::Result<()> {
+        let (pa, pb) = (a.full(rel), b.full(rel));
+        match (ea, eb) {
+            (Entry::Broken(e), _) => self.error(out, &format!("<{pa}> in <{}>: {e}", a.label)),
+            (_, Entry::Broken(e)) => self.error(out, &format!("<{pb}> in <{}>: {e}", b.label)),
+            (Entry::Soft(x), Entry::Soft(y)) => {
+                // Followed links that are left are dangling on both sides,
+                // which h5diff counts as the same.
+                if x != y && !self.opts.follow {
+                    self.diffs += 1;
+                    self.say(out, &format!("soft link: <{pa}> -> {x} and <{pb}> -> {y}"))?;
+                }
+                Ok(())
+            }
+            (Entry::External(f1, p1), Entry::External(f2, p2)) => {
+                if (f1, p1) != (f2, p2) {
+                    self.diffs += 1;
+                    self.say(
+                        out,
+                        &format!("external link: <{pa}> -> {f1}:{p1} and <{pb}> -> {f2}:{p2}"),
+                    )?;
+                }
+                Ok(())
+            }
+            (Entry::UserDefined(x), Entry::UserDefined(y)) => {
+                if x != y {
+                    self.diffs += 1;
+                    self.say(out, &format!("user-defined link: <{pa}> and <{pb}> differ"))?;
+                }
+                Ok(())
+            }
+            (Entry::Obj(aa, ka), Entry::Obj(ab, kb)) if ka == kb => {
+                let (ha, hb) = match (a.h5.header(*aa), b.h5.header(*ab)) {
+                    (Ok(x), Ok(y)) => (x, y),
+                    (Err(e), _) | (_, Err(e)) => return self.error(out, &format!("<{pa}>: {e}")),
+                };
+                let before = self.diffs;
+                let mut rows = Vec::new();
+                match ka {
+                    Kind::Dataset => self.dataset(out, a, b, &pa, &pb, &ha, &hb, &mut rows)?,
+                    Kind::Datatype => match (a.h5.datatype(&ha), b.h5.datatype(&hb)) {
+                        (Ok(x), Ok(y)) => {
+                            if x != y {
+                                self.diffs += 1;
+                                rows.push("datatypes differ".to_string());
+                            }
+                        }
+                        (Err(e), _) | (_, Err(e)) => self.error(out, &format!("<{pa}>: {e}"))?,
+                    },
+                    _ => {}
+                }
+                self.attributes(out, a, b, &pa, &ha, &hb, &mut rows)?;
+                let n = self.diffs - before;
+                if n > 0 && !self.opts.quiet {
+                    writeln!(out.o, "{}: <{pa}> and <{pb}>", kind_word(*ka))?;
+                    for r in &rows {
+                        writeln!(out.o, "{r}")?;
+                    }
+                    writeln!(out.o, "{n} difference(s) found")?;
+                    self.per_object += n;
+                }
+                Ok(())
+            }
+            _ => {
+                self.diffs += 1;
+                self.say(
+                    out,
+                    &format!(
+                        "Not comparable: <{pa}> is a {} and <{pb}> is a {}",
+                        entry_word(ea),
+                        entry_word(eb)
+                    ),
+                )
+            }
+        }
+    }
+
+    #[allow(clippy::too_many_arguments)]
+    fn dataset(
+        &mut self,
+        out: &mut Out,
+        a: &Side,
+        b: &Side,
+        pa: &str,
+        pb: &str,
+        ha: &ObjectHeader,
+        hb: &ObjectHeader,
+        rows: &mut Vec<String>,
+    ) -> std::io::Result<()> {
+        let got = (|| -> crate::h5::Result<_> {
+            Ok((
+                a.h5.datatype(ha)?,
+                a.h5.resolved_dataspace(pa, ha)?,
+                b.h5.datatype(hb)?,
+                b.h5.resolved_dataspace(pb, hb)?,
+            ))
+        })();
+        let (dta, dsa, dtb, dsb) = match got {
+            Ok(x) => x,
+            Err(e) => return self.error(out, &format!("<{pa}>: {e}")),
+        };
+        if let Some(why) = not_comparable(&dta, &dsa, &dtb, &dsb) {
+            self.diffs += 1;
+            rows.push(format!("Not comparable: {why}"));
+            return Ok(());
+        }
+        let raw =
+            a.h5.read_dataset(pa, &dta, &dsa)
+                .and_then(|x| Ok((x, b.h5.read_dataset(pb, &dtb, &dsb)?)));
+        let (ra, rb) = match raw {
+            Ok(x) => x,
+            Err(e) => return self.error(out, &format!("<{pa}>: {e}")),
+        };
+        self.values(out, a, b, pa, (&dta, &ra), (&dtb, &rb), &dsa, rows)
+    }
+
+    #[allow(clippy::too_many_arguments)]
+    fn values(
+        &mut self,
+        out: &mut Out,
+        a: &Side,
+        b: &Side,
+        what: &str,
+        (dta, ra): (&Datatype, &[u8]),
+        (dtb, rb): (&Datatype, &[u8]),
+        ds: &Dataspace,
+        rows: &mut Vec<String>,
+    ) -> std::io::Result<()> {
+        let n = crate::h5::num_elements(ds).unwrap_or(0) as usize;
+        let (da, db) = (Decoder::new(a.h5), Decoder::new(b.h5));
+        let dims: Vec<u64> = match ds.space_type {
+            DataspaceType::Simple => ds.dimensions.clone(),
+            _ => vec![1],
+        };
+        let mut found = 0u64;
+        let mut header = false;
+        for i in 0..n {
+            let va = da.element(dta, ra, i);
+            let vb = db.element(dtb, rb, i);
+            if let (Value::Error(e), _) | (_, Value::Error(e)) = (&va, &vb) {
+                return self.error(out, &format!("<{what}> element {i}: {e}"));
+            }
+            if self.equal(a, b, &va, &vb) {
+                continue;
+            }
+            found += 1;
+            if self.opts.report && (found as usize) <= self.opts.count {
+                if !header {
+                    header = true;
+                    rows.push(format!(
+                        "{:<24}{:<24}{:<24}{}",
+                        "position", "value 1", "value 2", "difference"
+                    ));
+                    rows.push("-".repeat(80));
+                }
+                let pos = format!("[ {} ]", index(i as u64, &dims));
+                let ta = value::text(&va, &|_| None);
+                let tb = value::text(&vb, &|_| None);
+                let dif = match (int_of(&va), int_of(&vb), number(&va), number(&vb)) {
+                    (Some(x), Some(y), ..) => x.abs_diff(y).to_string(),
+                    (_, _, Some(x), Some(y)) => value::fmt_float((x - y).abs(), 64),
+                    _ => String::new(),
+                };
+                rows.push(format!("{pos:<24}{ta:<24}{tb:<24}{dif}"));
+            }
+        }
+        self.diffs += found;
+        Ok(())
+    }
+
+    #[allow(clippy::too_many_arguments)]
+    fn attributes(
+        &mut self,
+        out: &mut Out,
+        a: &Side,
+        b: &Side,
+        path: &str,
+        ha: &ObjectHeader,
+        hb: &ObjectHeader,
+        rows: &mut Vec<String>,
+    ) -> std::io::Result<()> {
+        let (la, lb) = match (a.h5.attributes(ha), b.h5.attributes(hb)) {
+            (Ok(x), Ok(y)) => (x, y),
+            (Err(e), _) | (_, Err(e)) => return self.error(out, &format!("<{path}>: {e}")),
+        };
+        for e in la.1.iter().chain(lb.1.iter()) {
+            self.error(out, &format!("<{path}>: attribute: {e}"))?;
+        }
+        let ma: BTreeMap<&str, &AttributeMessage> =
+            la.0.iter().map(|x| (x.name.as_str(), x)).collect();
+        let mb: BTreeMap<&str, &AttributeMessage> =
+            lb.0.iter().map(|x| (x.name.as_str(), x)).collect();
+        let mut names: Vec<&str> = ma.keys().chain(mb.keys()).copied().collect();
+        names.sort_unstable();
+        names.dedup();
+        for n in names {
+            match (ma.get(n), mb.get(n)) {
+                (Some(x), Some(y)) => {
+                    if let Some(why) =
+                        not_comparable(&x.datatype, &x.dataspace, &y.datatype, &y.dataspace)
+                    {
+                        self.diffs += 1;
+                        rows.push(format!("attribute \"{n}\": not comparable: {why}"));
+                        continue;
+                    }
+                    for (m, h5) in [(x, a.h5), (y, b.h5)] {
+                        let need =
+                            crate::h5::byte_len(&m.dataspace, &m.datatype).unwrap_or(u64::MAX);
+                        if need > h5.max_bytes || (m.raw_data.len() as u64) < need {
+                            return self.error(
+                                out,
+                                &format!(
+                                    "<{path}> attribute \"{n}\": value is truncated or too large"
+                                ),
+                            );
+                        }
+                    }
+                    let before = rows.len();
+                    let what = format!("{path}\" attribute \"{n}");
+                    self.values(
+                        out,
+                        a,
+                        b,
+                        &what,
+                        (&x.datatype, &x.raw_data),
+                        (&y.datatype, &y.raw_data),
+                        &x.dataspace,
+                        rows,
+                    )?;
+                    if rows.len() > before {
+                        rows.insert(before, format!("attribute \"{n}\":"));
+                    }
+                }
+                (Some(_), None) => {
+                    self.diffs += 1;
+                    rows.push(format!("attribute \"{n}\" exists only in <{}>", a.label));
+                }
+                (None, Some(_)) => {
+                    self.diffs += 1;
+                    rows.push(format!("attribute \"{n}\" exists only in <{}>", b.label));
+                }
+                (None, None) => {}
+            }
+        }
+        Ok(())
+    }
+
+    fn equal(&self, a: &Side, b: &Side, x: &Value, y: &Value) -> bool {
+        // Integers in integer arithmetic: through f64 they lose precision
+        // above 2^53, and values that differ would compare equal.
+        if let (Some(p), Some(q)) = (int_of(x), int_of(y)) {
+            return int_close(p, q, self.opts.tol);
+        }
+        if let (Some(p), Some(q)) = (number(x), number(y)) {
+            return close(p, q, self.opts.tol);
+        }
+        match (x, y) {
+            (Value::Str(p), Value::Str(q)) => p == q,
+            (Value::Bytes(p), Value::Bytes(q)) | (Value::OtherRef(p), Value::OtherRef(q)) => p == q,
+            (Value::Compound(p), Value::Compound(q)) => {
+                p.len() == q.len()
+                    && p.iter()
+                        .zip(q)
+                        .all(|((_, u), (_, v))| self.equal(a, b, u, v))
+            }
+            (Value::Array(p), Value::Array(q)) | (Value::Seq(p), Value::Seq(q)) => {
+                p.len() == q.len() && p.iter().zip(q).all(|(u, v)| self.equal(a, b, u, v))
+            }
+            (Value::Ref(None), Value::Ref(None)) => true,
+            (Value::Ref(Some(p)), Value::Ref(Some(q))) => {
+                // Addresses mean nothing across files: compare the paths the
+                // references lead to.
+                let (pp, qq) = (a.rel_paths().get(p), b.rel_paths().get(q));
+                pp.is_some() && pp == qq
+            }
+            _ => false,
+        }
+    }
+}
+
+fn int_of(v: &Value) -> Option<i128> {
+    match v {
+        Value::Int(i) | Value::Enum(_, i) => Some(*i),
+        _ => None,
+    }
+}
+
+fn number(v: &Value) -> Option<f64> {
+    match v {
+        Value::Int(i) | Value::Enum(_, i) => Some(*i as f64),
+        Value::Float(f, _) => Some(*f),
+        _ => None,
+    }
+}
+
+fn close(a: f64, b: f64, tol: Tol) -> bool {
+    if a.is_nan() || b.is_nan() {
+        return a.is_nan() && b.is_nan();
+    }
+    if a == b {
+        return true;
+    }
+    let d = (a - b).abs();
+    match tol {
+        Tol::Exact => false,
+        Tol::Delta(t) => d <= t,
+        Tol::Relative(r) => a != 0.0 && d / a.abs() <= r,
+    }
+}
+
+/// `close` for integers, exactly: the difference is taken in i128, so no
+/// precision is lost at any 64-bit magnitude.
+fn int_close(a: i128, b: i128, tol: Tol) -> bool {
+    if a == b {
+        return true;
+    }
+    let d = a.abs_diff(b);
+    match tol {
+        Tol::Exact => false,
+        // d is whole, so d <= t exactly when d <= floor(t) (saturating).
+        Tol::Delta(t) => d <= t.floor() as u128,
+        // d >= 1 here, so the quotient is never rounded to 0.
+        Tol::Relative(r) => a != 0 && d as f64 / a.unsigned_abs() as f64 <= r,
+    }
+}
+
+fn entry_word(e: &Entry) -> &'static str {
+    match e {
+        Entry::Obj(_, k) => kind_word(*k),
+        Entry::Soft(_) => "soft link",
+        Entry::External(..) => "external link",
+        Entry::UserDefined(_) => "user-defined link",
+        Entry::Broken(_) => "unreadable object",
+    }
+}
+
+/// Why two datasets/attributes cannot be compared element by element.
+fn not_comparable(
+    dta: &Datatype,
+    dsa: &Dataspace,
+    dtb: &Datatype,
+    dsb: &Dataspace,
+) -> Option<String> {
+    let rank = |d: &Dataspace| match d.space_type {
+        DataspaceType::Simple => Some(d.dimensions.clone()),
+        DataspaceType::Scalar => Some(Vec::new()),
+        DataspaceType::Null => None,
+    };
+    let (sa, sb) = (rank(dsa), rank(dsb));
+    if sa != sb {
+        let show = |s: &Option<Vec<u64>>| match s {
+            None => "null".to_string(),
+            Some(d) if d.is_empty() => "scalar".to_string(),
+            Some(d) => format!("{d:?}"),
+        };
+        return Some(format!("shapes differ: {} and {}", show(&sa), show(&sb)));
+    }
+    if !types_comparable(dta, dtb) {
+        return Some(format!(
+            "datatypes differ: {} and {}",
+            crate::dtype::short(dta),
+            crate::dtype::short(dtb)
+        ));
+    }
+    None
+}
+
+fn types_comparable(a: &Datatype, b: &Datatype) -> bool {
+    use crate::dtype::class;
+    let numeric = |t: &Datatype| matches!(class(t), "integer" | "float");
+    if numeric(a) && numeric(b) {
+        return class(a) == class(b);
+    }
+    if class(a) != class(b) {
+        return false;
+    }
+    match (a, b) {
+        (Datatype::Compound { members: ma, .. }, Datatype::Compound { members: mb, .. }) => {
+            ma.len() == mb.len()
+                && ma
+                    .iter()
+                    .zip(mb)
+                    .all(|(x, y)| x.name == y.name && types_comparable(&x.datatype, &y.datatype))
+        }
+        (
+            Datatype::Array {
+                base_type: x,
+                dimensions: dx,
+            },
+            Datatype::Array {
+                base_type: y,
+                dimensions: dy,
+            },
+        ) => dx == dy && types_comparable(x, y),
+        (
+            Datatype::VariableLength { base_type: x, .. },
+            Datatype::VariableLength { base_type: y, .. },
+        ) => class(a) == "string" || types_comparable(x, y),
+        (
+            Datatype::Enumeration { base_type: x, .. },
+            Datatype::Enumeration { base_type: y, .. },
+        ) => types_comparable(x, y),
+        _ => true,
+    }
+}
+
+fn index(mut i: u64, dims: &[u64]) -> String {
+    let mut idx = vec![0u64; dims.len()];
+    for (k, &d) in dims.iter().enumerate().rev() {
+        let d = d.max(1);
+        idx[k] = i % d;
+        i /= d;
+    }
+    idx.iter()
+        .map(|x| x.to_string())
+        .collect::<Vec<_>>()
+        .join(" ")
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn tolerances() {
+        assert!(close(1.0, 1.0, Tol::Exact));
+        assert!(!close(1.0, 1.001, Tol::Exact));
+        assert!(close(1.0, 1.001, Tol::Delta(0.01)));
+        assert!(!close(1.0, 1.1, Tol::Delta(0.01)));
+        assert!(close(100.0, 101.0, Tol::Relative(0.02)));
+        assert!(!close(0.0, 1e-9, Tol::Relative(0.5)));
+        assert!(close(f64::NAN, f64::NAN, Tol::Exact));
+        assert!(!close(f64::NAN, 1.0, Tol::Delta(1e9)));
+    }
+
+    #[test]
+    fn integer_tolerances_are_exact_above_2_pow_53() {
+        let big = 1i128 << 60;
+        assert!(!int_close(big, big + 1, Tol::Delta(0.0)));
+        assert!(!int_close(big, big + 1, Tol::Delta(0.5)));
+        assert!(int_close(big, big + 1, Tol::Delta(1.0)));
+        assert!(!int_close(big, big + 200, Tol::Delta(199.9)));
+        assert!(!int_close(big, big + 1, Tol::Relative(0.0)));
+        assert!(!int_close(big, big + 1, Tol::Relative(1e-19)));
+        assert!(int_close(big, big + 1, Tol::Relative(1e-18)));
+        let umax = i128::from(u64::MAX);
+        assert!(!int_close(umax, umax - 1, Tol::Delta(0.0)));
+        assert!(int_close(
+            i128::from(i64::MIN),
+            umax,
+            Tol::Delta(f64::INFINITY)
+        ));
+        assert!(!int_close(0, 1, Tol::Relative(1e9)));
+    }
+
+    #[test]
+    fn positions() {
+        assert_eq!(index(5, &[3, 4]), "1 1");
+        assert_eq!(index(0, &[1]), "0");
+    }
+}

crates/clawhdf5-tools/src/dtype.rs

@@ -0,0 +1,529 @@
+//! Names for datatypes: a short one for listings, h5ls's long form, h5dump's
+//! DDL and the HDF Group's hdf5-json type objects.
+
+use clawhdf5_format::datatype::{
+    CharacterSet, Datatype, DatatypeByteOrder, ReferenceType, StringPadding,
+};
+use serde_json::{Value as J, json};
+
+fn be(o: &DatatypeByteOrder) -> bool {
+    matches!(o, DatatypeByteOrder::BigEndian)
+}
+
+fn order_suffix(o: &DatatypeByteOrder) -> &'static str {
+    match o {
+        DatatypeByteOrder::LittleEndian => "LE",
+        DatatypeByteOrder::BigEndian => "BE",
+        DatatypeByteOrder::Vax => "VAX",
+    }
+}
+
+fn order_word(o: &DatatypeByteOrder) -> &'static str {
+    match o {
+        DatatypeByteOrder::LittleEndian => "little-endian",
+        DatatypeByteOrder::BigEndian => "big-endian",
+        DatatypeByteOrder::Vax => "VAX-order",
+    }
+}
+
+/// True when a float is laid out exactly as IEEE 754 binary16/32/64.
+pub fn is_ieee(dt: &Datatype) -> bool {
+    let Datatype::FloatingPoint {
+        size,
+        bit_offset,
+        bit_precision,
+        exponent_location,
+        exponent_size,
+        mantissa_location,
+        mantissa_size,
+        exponent_bias,
+        ..
+    } = dt
+    else {
+        return false;
+    };
+    let std = match size {
+        2 => (16, 10, 5, 10, 15),
+        4 => (32, 23, 8, 23, 127),
+        8 => (64, 52, 11, 52, 1023),
+        _ => return false,
+    };
+    *bit_offset == 0
+        && *mantissa_location == 0
+        && (
+            *bit_precision,
+            *exponent_location,
+            *exponent_size,
+            *mantissa_size,
+            *exponent_bias,
+        ) == std
+}
+
+/// Short name used by `ls`: `int32`, `float64-be`, `string[3]`, ...
+pub fn short(dt: &Datatype) -> String {
+    match dt {
+        Datatype::FixedPoint {
+            size,
+            signed,
+            byte_order,
+            ..
+        } => format!(
+            "{}int{}{}",
+            if *signed { "" } else { "u" },
+            u64::from(*size) * 8,
+            if be(byte_order) { "-be" } else { "" }
+        ),
+        Datatype::FloatingPoint {
+            size, byte_order, ..
+        } => format!(
+            "float{}{}",
+            u64::from(*size) * 8,
+            if be(byte_order) { "-be" } else { "" }
+        ),
+        Datatype::Time { size, .. } => format!("time{}", u64::from(*size) * 8),
+        Datatype::String { size, .. } => format!("string[{size}]"),
+        Datatype::BitField { size, .. } => format!("bitfield{}", u64::from(*size) * 8),
+        Datatype::Opaque { size, .. } => format!("opaque[{size}]"),
+        Datatype::Compound { members, .. } => format!(
+            "compound{{{}}}",
+            members
+                .iter()
+                .map(|m| format!("{}: {}", m.name, short(&m.datatype)))
+                .collect::<Vec<_>>()
+                .join(", ")
+        ),
+        Datatype::Reference { ref_type, .. } => match ref_type {
+            ReferenceType::Object | ReferenceType::Object2 => "object-reference".into(),
+            ReferenceType::DatasetRegion | ReferenceType::DatasetRegion2 => {
+                "region-reference".into()
+            }
+            ReferenceType::Attribute => "attribute-reference".into(),
+        },
+        Datatype::Enumeration { base_type, .. } => format!("enum<{}>", short(base_type)),
+        Datatype::VariableLength {
+            is_string: true, ..
+        } => "vlen-string".into(),
+        Datatype::VariableLength { base_type, .. } => format!("vlen<{}>", short(base_type)),
+        Datatype::Array {
+            base_type,
+            dimensions,
+        } => format!(
+            "array[{}]<{}>",
+            dimensions
+                .iter()
+                .map(|d| d.to_string())
+                .collect::<Vec<_>>()
+                .join(","),
+            short(base_type)
+        ),
+    }
+}
+
+fn pad_word(p: &StringPadding) -> &'static str {
+    match p {
+        StringPadding::NullTerminate => "null-terminated",
+        StringPadding::NullPad => "null-padded",
+        StringPadding::SpacePad => "space-padded",
+    }
+}
+
+fn cset_word(c: &CharacterSet) -> &'static str {
+    match c {
+        CharacterSet::Ascii => "ASCII",
+        CharacterSet::Utf8 => "UTF-8",
+    }
+}
+
+/// h5ls -v style description.
+pub fn long(dt: &Datatype) -> String {
+    match dt {
+        Datatype::FixedPoint {
+            size,
+            signed,
+            byte_order,
+            ..
+        } => format!(
+            "{}-bit {} {}integer",
+            u64::from(*size) * 8,
+            order_word(byte_order),
+            if *signed { "" } else { "unsigned " }
+        ),
+        Datatype::FloatingPoint {
+            size, byte_order, ..
+        } => {
+            if is_ieee(dt) {
+                format!(
+                    "IEEE {}-bit {} float",
+                    u64::from(*size) * 8,
+                    order_word(byte_order)
+                )
+            } else {
+                format!(
+                    "{}-bit {} non-IEEE float",
+                    u64::from(*size) * 8,
+                    order_word(byte_order)
+                )
+            }
+        }
+        Datatype::Time { size, .. } => format!("{}-bit time", u64::from(*size) * 8),
+        Datatype::String {
+            size,
+            padding,
+            charset,
+        } => format!(
+            "{size}-byte {} {} string",
+            pad_word(padding),
+            cset_word(charset)
+        ),
+        Datatype::BitField {
+            size, byte_order, ..
+        } => format!(
+            "{}-bit {} bitfield",
+            u64::from(*size) * 8,
+            order_word(byte_order)
+        ),
+        Datatype::Opaque { size, tag } => {
+            let tag = String::from_utf8_lossy(tag);
+            let tag = tag.trim_end_matches('\0');
+            if tag.is_empty() {
+                format!("{size}-byte opaque type")
+            } else {
+                format!("{size}-byte opaque type (tag \"{tag}\")")
+            }
+        }
+        Datatype::Compound { size, members } => {
+            let mut s = String::from("struct {");
+            for m in members {
+                s.push_str(&format!(
+                    "\n                   \"{}\" +{} {}",
+                    m.name,
+                    m.byte_offset,
+                    long(&m.datatype)
+                ));
+            }
+            s.push_str(&format!("\n               }} {size} bytes"));
+            s
+        }
+        Datatype::Reference { ref_type, .. } => match ref_type {
+            ReferenceType::Object | ReferenceType::Object2 => "object reference".into(),
+            ReferenceType::DatasetRegion | ReferenceType::DatasetRegion2 => {
+                "dataset region reference".into()
+            }
+            ReferenceType::Attribute => "attribute reference".into(),
+        },
+        Datatype::Enumeration {
+            base_type, members, ..
+        } => {
+            let vals: Vec<String> = members
+                .iter()
+                .map(|m| format!("{} = {}", m.name, enum_value_text(base_type, &m.value)))
+                .collect();
+            format!("enum {} {{{}}}", long(base_type), vals.join(", "))
+        }
+        Datatype::VariableLength {
+            is_string: true,
+            padding,
+            charset,
+            ..
+        } => format!(
+            "variable-length {} {} string",
+            padding.as_ref().map(pad_word).unwrap_or("null-terminated"),
+            charset.as_ref().map(cset_word).unwrap_or("ASCII")
+        ),
+        Datatype::VariableLength { base_type, .. } => {
+            format!("variable length of {}", long(base_type))
+        }
+        Datatype::Array {
+            base_type,
+            dimensions,
+        } => format!(
+            "[{}] {}",
+            dimensions
+                .iter()
+                .map(|d| d.to_string())
+                .collect::<Vec<_>>()
+                .join(","),
+            long(base_type)
+        ),
+    }
+}
+
+/// The integer an enum member's value bytes hold, as text.
+pub fn enum_value_text(base: &Datatype, bytes: &[u8]) -> String {
+    match crate::value::decode_int(base, bytes) {
+        Some(v) => v.to_string(),
+        None => crate::value::hex(bytes),
+    }
+}
+
+/// h5dump's predefined name for an atomic type, if it has one.
+fn atomic_ddl(dt: &Datatype) -> Option<String> {
+    match dt {
+        Datatype::FixedPoint {
+            size,
+            signed,
+            byte_order,
+            ..
+        } => Some(format!(
+            "H5T_STD_{}{}{}",
+            if *signed { "I" } else { "U" },
+            u64::from(*size) * 8,
+            order_suffix(byte_order)
+        )),
+        Datatype::FloatingPoint {
+            size, byte_order, ..
+        } if is_ieee(dt) => Some(format!(
+            "H5T_IEEE_F{}{}",
+            u64::from(*size) * 8,
+            order_suffix(byte_order)
+        )),
+        Datatype::BitField {
+            size, byte_order, ..
+        } => Some(format!(
+            "H5T_STD_B{}{}",
+            u64::from(*size) * 8,
+            order_suffix(byte_order)
+        )),
+        Datatype::Time { .. } => Some("H5T_TIME".into()),
+        _ => None,
+    }
+}
+
+/// h5dump DDL for a datatype; `ind` is the indentation of the line the type
+/// starts on (continuation lines are indented relative to it).
+pub fn ddl(dt: &Datatype, ind: usize) -> String {
+    let pad = " ".repeat(ind + 3);
+    let end = " ".repeat(ind);
+    if let Some(s) = atomic_ddl(dt) {
+        return s;
+    }
+    match dt {
+        Datatype::FloatingPoint {
+            size,
+            byte_order,
+            bit_offset,
+            bit_precision,
+            exponent_location,
+            exponent_size,
+            mantissa_location,
+            mantissa_size,
+            exponent_bias,
+        } => format!(
+            "H5T_FLOAT {{\n{pad}SIZE {size};\n{pad}ORDER H5T_ORDER_{};\n{pad}OFFSET {bit_offset};\n\
+             {pad}PRECISION {bit_precision};\n{pad}EXPONENT {exponent_location} {exponent_size} \
+             BIAS {exponent_bias};\n{pad}MANTISSA {mantissa_location} {mantissa_size};\n{end}}}",
+            order_suffix(byte_order)
+        ),
+        Datatype::String {
+            size,
+            padding,
+            charset,
+        } => string_ddl(&size.to_string(), padding, charset, &pad, &end),
+        Datatype::VariableLength {
+            is_string: true,
+            padding,
+            charset,
+            ..
+        } => string_ddl(
+            "H5T_VARIABLE",
+            padding.as_ref().unwrap_or(&StringPadding::NullTerminate),
+            charset.as_ref().unwrap_or(&CharacterSet::Ascii),
+            &pad,
+            &end,
+        ),
+        Datatype::VariableLength { base_type, .. } => {
+            format!("H5T_VLEN {{ {} }}", ddl(base_type, ind))
+        }
+        Datatype::Opaque { size, tag } => {
+            let tag = String::from_utf8_lossy(tag);
+            format!(
+                "H5T_OPAQUE {{\n{pad}OPAQUE_SIZE {size};\n{pad}OPAQUE_TAG \"{}\";\n{end}}}",
+                tag.trim_end_matches('\0')
+            )
+        }
+        Datatype::Compound { members, .. } => {
+            let mut s = String::from("H5T_COMPOUND {\n");
+            for m in members {
+                s.push_str(&format!(
+                    "{pad}{} \"{}\";\n",
+                    ddl(&m.datatype, ind + 3),
+                    m.name
+                ));
+            }
+            s.push_str(&end);
+            s.push('}');
+            s
+        }
+        Datatype::Reference { ref_type, .. } => match ref_type {
+            ReferenceType::Object => "H5T_REFERENCE { H5T_STD_REF_OBJECT }".into(),
+            ReferenceType::DatasetRegion => "H5T_REFERENCE { H5T_STD_REF_DSETREG }".into(),
+            _ => "H5T_REFERENCE { H5T_STD_REF }".into(),
+        },
+        Datatype::Enumeration {
+            base_type, members, ..
+        } => {
+            let mut s = format!("H5T_ENUM {{\n{pad}{};\n", ddl(base_type, ind + 3));
+            for m in members {
+                let name = format!("\"{}\"", m.name);
+                s.push_str(&format!(
+                    "{pad}{name:<18} {};\n",
+                    enum_value_text(base_type, &m.value)
+                ));
+            }
+            s.push_str(&end);
+            s.push('}');
+            s
+        }
+        Datatype::Array {
+            base_type,
+            dimensions,
+        } => format!(
+            "H5T_ARRAY {{ {} {} }}",
+            dimensions
+                .iter()
+                .map(|d| format!("[{d}]"))
+                .collect::<String>(),
+            ddl(base_type, ind)
+        ),
+        // Atomic types were handled above.
+        _ => short(dt),
+    }
+}
+
+fn string_ddl(
+    size: &str,
+    padding: &StringPadding,
+    charset: &CharacterSet,
+    pad: &str,
+    end: &str,
+) -> String {
+    let p = match padding {
+        StringPadding::NullTerminate => "H5T_STR_NULLTERM",
+        StringPadding::NullPad => "H5T_STR_NULLPAD",
+        StringPadding::SpacePad => "H5T_STR_SPACEPAD",
+    };
+    let c = match charset {
+        CharacterSet::Ascii => "H5T_CSET_ASCII",
+        CharacterSet::Utf8 => "H5T_CSET_UTF8",
+    };
+    format!(
+        "H5T_STRING {{\n{pad}STRSIZE {size};\n{pad}STRPAD {p};\n{pad}CSET {c};\n{pad}CTYPE H5T_C_S1;\n{end}}}"
+    )
+}
+
+/// hdf5-json type object.
+pub fn json(dt: &Datatype) -> J {
+    match dt {
+        Datatype::FixedPoint { .. } => {
+            json!({"class": "H5T_INTEGER", "base": atomic_ddl(dt)})
+        }
+        Datatype::FloatingPoint { .. } if is_ieee(dt) => {
+            json!({"class": "H5T_FLOAT", "base": atomic_ddl(dt)})
+        }
+        Datatype::FloatingPoint {
+            size, byte_order, ..
+        } => json!({
+            "class": "H5T_FLOAT",
+            "size": size,
+            "order": format!("H5T_ORDER_{}", order_suffix(byte_order)),
+        }),
+        Datatype::BitField { .. } => json!({"class": "H5T_BITFIELD", "base": atomic_ddl(dt)}),
+        Datatype::Time { size, .. } => json!({"class": "H5T_TIME", "size": size}),
+        Datatype::String {
+            size,
+            padding,
+            charset,
+        } => json!({
+            "class": "H5T_STRING",
+            "charSet": cset_json(charset),
+            "strPad": pad_json(padding),
+            "length": size,
+        }),
+        Datatype::VariableLength {
+            is_string: true,
+            padding,
+            charset,
+            ..
+        } => json!({
+            "class": "H5T_STRING",
+            "charSet": cset_json(charset.as_ref().unwrap_or(&CharacterSet::Ascii)),
+            "strPad": pad_json(padding.as_ref().unwrap_or(&StringPadding::NullTerminate)),
+            "length": "H5T_VARIABLE",
+        }),
+        Datatype::VariableLength { base_type, .. } => {
+            json!({"class": "H5T_VLEN", "base": json(base_type)})
+        }
+        Datatype::Opaque { size, tag } => json!({
+            "class": "H5T_OPAQUE",
+            "size": size,
+            "tag": String::from_utf8_lossy(tag).trim_end_matches('\0'),
+        }),
+        Datatype::Compound { members, .. } => json!({
+            "class": "H5T_COMPOUND",
+            "fields": members
+                .iter()
+                .map(|m| json!({"name": m.name, "type": json(&m.datatype)}))
+                .collect::<Vec<_>>(),
+        }),
+        Datatype::Reference { ref_type, .. } => json!({
+            "class": "H5T_REFERENCE",
+            "base": match ref_type {
+                ReferenceType::Object => "H5T_STD_REF_OBJ",
+                ReferenceType::DatasetRegion => "H5T_STD_REF_DSETREG",
+                _ => "H5T_STD_REF",
+            },
+        }),
+        Datatype::Enumeration {
+            base_type, members, ..
+        } => {
+            let mut mapping = serde_json::Map::new();
+            for m in members {
+                let v = crate::value::decode_int(base_type, &m.value)
+                    .and_then(|v| i64::try_from(v).ok())
+                    .map(J::from)
+                    .unwrap_or_else(|| J::from(crate::value::hex(&m.value)));
+                mapping.insert(m.name.clone(), v);
+            }
+            json!({"class": "H5T_ENUM", "base": json(base_type), "mapping": mapping})
+        }
+        Datatype::Array {
+            base_type,
+            dimensions,
+        } => json!({"class": "H5T_ARRAY", "base": json(base_type), "dims": dimensions}),
+    }
+}
+
+fn cset_json(c: &CharacterSet) -> &'static str {
+    match c {
+        CharacterSet::Ascii => "H5T_CSET_ASCII",
+        CharacterSet::Utf8 => "H5T_CSET_UTF8",
+    }
+}
+
+fn pad_json(p: &StringPadding) -> &'static str {
+    match p {
+        StringPadding::NullTerminate => "H5T_STR_NULLTERM",
+        StringPadding::NullPad => "H5T_STR_NULLPAD",
+        StringPadding::SpacePad => "H5T_STR_SPACEPAD",
+    }
+}
+
+/// Class name used to decide whether two datatypes can be compared.
+pub fn class(dt: &Datatype) -> &'static str {
+    match dt {
+        Datatype::FixedPoint { .. } => "integer",
+        Datatype::FloatingPoint { .. } => "float",
+        Datatype::Time { .. } => "time",
+        Datatype::String { .. } => "string",
+        Datatype::BitField { .. } => "bitfield",
+        Datatype::Opaque { .. } => "opaque",
+        Datatype::Compound { .. } => "compound",
+        Datatype::Reference { .. } => "reference",
+        Datatype::Enumeration { .. } => "enum",
+        Datatype::VariableLength {
+            is_string: true, ..
+        } => "string",
+        Datatype::VariableLength { .. } => "vlen",
+        Datatype::Array { .. } => "array",
+    }
+}

crates/clawhdf5-tools/src/h5.rs

@@ -0,0 +1,718 @@
+//! The file model the tools share: an open file, its objects, their links
+//! and the messages that describe a dataset, all read through
+//! `clawhdf5-format` (and the `clawhdf5` facade for dataset values).
+//!
+//! Nothing here trusts the file: every size is checked before it is used,
+//! and every failure is an [`Error`] carrying the address it happened at.
+
+use std::cell::RefCell;
+use std::collections::HashMap;
+use std::path::{Path, PathBuf};
+use std::rc::Rc;
+
+use clawhdf5::File;
+use clawhdf5_format::attribute::{AttributeMessage, extract_attributes_tolerant};
+use clawhdf5_format::attribute_info::AttributeInfoMessage;
+use clawhdf5_format::btree_v2::{BTreeV2Header, collect_btree_v2_records};
+use clawhdf5_format::data_layout::DataLayout;
+use clawhdf5_format::dataspace::{Dataspace, DataspaceType};
+use clawhdf5_format::datatype::Datatype;
+use clawhdf5_format::error::FormatError;
+use clawhdf5_format::filter_pipeline::FilterPipeline;
+use clawhdf5_format::fractal_heap::FractalHeapHeader;
+use clawhdf5_format::global_heap::GlobalHeapCollection;
+use clawhdf5_format::group_v1;
+use clawhdf5_format::link_info::LinkInfoMessage;
+use clawhdf5_format::link_message::{LinkMessage, LinkTarget};
+use clawhdf5_format::message_type::MessageType;
+use clawhdf5_format::object_header::ObjectHeader;
+use clawhdf5_format::superblock::Superblock;
+use clawhdf5_format::symbol_table::SymbolTableMessage;
+
+/// Largest dataset or attribute (in bytes) the tools will decode by default.
+/// A corrupt dataspace can claim far more elements than the file holds; the
+/// limit keeps such a file from exhausting memory. `--max-bytes` changes it.
+pub const DEFAULT_MAX_BYTES: u64 = 1 << 30;
+
+/// Upper bound on the objects one traversal visits, so a crafted file with
+/// millions of links cannot make a walk run for ever.
+pub const MAX_OBJECTS: usize = 1_000_000;
+
+/// A tool error: what went wrong and, when known, the file address (relative
+/// to the superblock, as every HDF5 address is) of the structure involved.
+#[derive(Debug, Clone)]
+pub struct Error {
+    pub addr: Option<u64>,
+    pub msg: String,
+    pub kind: ErrorKind,
+}
+
+/// Whether an error says something is wrong with the file.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum ErrorKind {
+    /// The file is damaged or not valid HDF5 (as far as clawhdf5 knows).
+    Corrupt,
+    /// Valid HDF5 that clawhdf5 cannot decode (a filter it does not
+    /// implement, external raw data files, ...).
+    Unsupported,
+    /// Refused by a limit of the tool (`--max-bytes`).
+    Limit,
+}
+
+impl Error {
+    pub fn new(msg: impl Into<String>) -> Self {
+        Self {
+            addr: None,
+            msg: msg.into(),
+            kind: ErrorKind::Corrupt,
+        }
+    }
+
+    pub fn at(addr: u64, msg: impl Into<String>) -> Self {
+        Self {
+            addr: Some(addr),
+            msg: msg.into(),
+            kind: ErrorKind::Corrupt,
+        }
+    }
+
+    pub fn with_kind(mut self, kind: ErrorKind) -> Self {
+        self.kind = kind;
+        self
+    }
+
+    /// The same error with `prefix: ` in front of its message.
+    pub fn context(mut self, prefix: &str) -> Self {
+        self.msg = format!("{prefix}: {}", self.msg);
+        self
+    }
+}
+
+fn format_error_kind(e: &FormatError) -> ErrorKind {
+    match e {
+        FormatError::UnsupportedFilter(_)
+        | FormatError::ExternalDataFilesUnsupported
+        | FormatError::ExternalLinkUnsupported { .. } => ErrorKind::Unsupported,
+        _ => ErrorKind::Corrupt,
+    }
+}
+
+impl std::fmt::Display for Error {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self.addr {
+            Some(a) => write!(f, "{} (at address {a:#x})", self.msg),
+            None => write!(f, "{}", self.msg),
+        }
+    }
+}
+
+impl From<FormatError> for Error {
+    fn from(e: FormatError) -> Self {
+        Error::new(e.to_string()).with_kind(format_error_kind(&e))
+    }
+}
+
+impl From<clawhdf5::Error> for Error {
+    fn from(e: clawhdf5::Error) -> Self {
+        let kind = match &e {
+            clawhdf5::Error::Format(f) => format_error_kind(f),
+            _ => ErrorKind::Corrupt,
+        };
+        Error::new(e.to_string()).with_kind(kind)
+    }
+}
+
+pub type Result<T> = std::result::Result<T, Error>;
+
+/// Attach an address to a format error.
+pub fn fe_at(addr: u64) -> impl Fn(FormatError) -> Error {
+    move |e| Error::at(addr, e.to_string())
+}
+
+/// What an object header describes.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Kind {
+    Group,
+    Dataset,
+    Datatype,
+    Unknown,
+}
+
+impl Kind {
+    pub fn of(h: &ObjectHeader) -> Kind {
+        let has = |t: MessageType| h.messages.iter().any(|m| m.msg_type == t);
+        if has(MessageType::DataLayout) {
+            Kind::Dataset
+        } else if has(MessageType::SymbolTable)
+            || has(MessageType::LinkInfo)
+            || has(MessageType::Link)
+            || has(MessageType::GroupInfo)
+        {
+            Kind::Group
+        } else if has(MessageType::Datatype) {
+            Kind::Datatype
+        } else {
+            Kind::Unknown
+        }
+    }
+
+    pub fn name(self) -> &'static str {
+        match self {
+            Kind::Group => "Group",
+            Kind::Dataset => "Dataset",
+            Kind::Datatype => "Type",
+            Kind::Unknown => "Unknown",
+        }
+    }
+}
+
+/// Where a link points.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum LinkKind {
+    Hard(u64),
+    Soft(String),
+    External {
+        file: String,
+        path: String,
+    },
+    /// A user-defined link class (type 65-255): its target means something
+    /// only to the application that registered the class.
+    UserDefined(u8),
+}
+
+#[derive(Debug, Clone)]
+pub struct Link {
+    pub name: String,
+    pub kind: LinkKind,
+}
+
+/// An open file.
+pub struct H5 {
+    pub path: PathBuf,
+    pub file: File,
+    pub max_bytes: u64,
+    heaps: RefCell<HashMap<u64, std::result::Result<Rc<GlobalHeapCollection>, String>>>,
+    /// Fractal heaps whose blocks were verified: `None` = sound.
+    verified_heaps: RefCell<HashMap<u64, Option<Error>>>,
+}
+
+impl H5 {
+    pub fn open(path: &Path) -> Result<H5> {
+        if !path.exists() {
+            return Err(Error::new(format!("{}: no such file", path.display())));
+        }
+        let file = File::open(path).map_err(|e| {
+            Error::new(format!(
+                "{}: not an HDF5 file this tool can open: {e}",
+                path.display()
+            ))
+        })?;
+        Ok(H5 {
+            path: path.to_path_buf(),
+            file,
+            max_bytes: DEFAULT_MAX_BYTES,
+            heaps: RefCell::new(HashMap::new()),
+            verified_heaps: RefCell::new(HashMap::new()),
+        })
+    }
+
+    /// The file's bytes from the superblock on: what every address indexes.
+    pub fn data(&self) -> &[u8] {
+        self.file.as_bytes()
+    }
+
+    pub fn sb(&self) -> &Superblock {
+        self.file.superblock()
+    }
+
+    pub fn os(&self) -> u8 {
+        self.sb().offset_size
+    }
+
+    pub fn ls(&self) -> u8 {
+        self.sb().length_size
+    }
+
+    pub fn root(&self) -> u64 {
+        self.sb().root_group_address
+    }
+
+    pub fn header(&self, addr: u64) -> Result<ObjectHeader> {
+        let off = usize::try_from(addr).map_err(|_| Error::at(addr, "address out of range"))?;
+        ObjectHeader::parse(self.data(), off, self.os(), self.ls())
+            .map_err(|e| Error::at(addr, format!("object header: {e}")))
+    }
+
+    /// The payload of the first message of type `t`, resolving a shared
+    /// message to the message it points at.
+    pub fn payload(&self, h: &ObjectHeader, t: MessageType) -> Result<Option<Vec<u8>>> {
+        match h.messages.iter().find(|m| m.msg_type == t) {
+            None => Ok(None),
+            Some(m) => {
+                clawhdf5_format::shared_message::message_data(self.data(), m, self.os(), self.ls())
+                    .map(|c| Some(c.into_owned()))
+                    .map_err(|e| Error::new(format!("{t:?} message: {e}")))
+            }
+        }
+    }
+
+    pub fn datatype(&self, h: &ObjectHeader) -> Result<Datatype> {
+        let b = self
+            .payload(h, MessageType::Datatype)?
+            .ok_or_else(|| Error::new("no datatype message"))?;
+        Datatype::parse(&b)
+            .map(|(d, _)| d)
+            .map_err(|e| Error::new(format!("datatype message: {e}")))
+    }
+
+    pub fn dataspace(&self, h: &ObjectHeader) -> Result<Dataspace> {
+        let b = self
+            .payload(h, MessageType::Dataspace)?
+            .ok_or_else(|| Error::new("no dataspace message"))?;
+        Dataspace::parse(&b, self.ls()).map_err(|e| Error::new(format!("dataspace message: {e}")))
+    }
+
+    pub fn layout(&self, h: &ObjectHeader) -> Result<DataLayout> {
+        let m = h
+            .messages
+            .iter()
+            .find(|m| m.msg_type == MessageType::DataLayout)
+            .ok_or_else(|| Error::new("no layout message"))?;
+        DataLayout::parse(&m.data, self.os(), self.ls())
+            .map_err(|e| Error::new(format!("layout message: {e}")))
+    }
+
+    pub fn filters(&self, h: &ObjectHeader) -> Result<Option<FilterPipeline>> {
+        match self.payload(h, MessageType::FilterPipeline)? {
+            None => Ok(None),
+            Some(b) => FilterPipeline::parse(&b)
+                .map(Some)
+                .map_err(|e| Error::new(format!("filter pipeline message: {e}"))),
+        }
+    }
+
+    /// Every attribute that can be read, plus one error per attribute that
+    /// cannot.
+    pub fn attributes(&self, h: &ObjectHeader) -> Result<(Vec<AttributeMessage>, Vec<String>)> {
+        if let Some(m) = h
+            .messages
+            .iter()
+            .find(|m| m.msg_type == MessageType::AttributeInfo)
+            && let Ok(ai) = AttributeInfoMessage::parse(&m.data, self.os())
+            && let Some(fh) = ai.fractal_heap_address
+        {
+            self.verified_heap(fh)
+                .map_err(|e| e.context("dense attribute storage"))?;
+        }
+        let (mut attrs, errs) = extract_attributes_tolerant(self.data(), h, self.os(), self.ls())
+            .map_err(|e| Error::new(format!("attributes: {e}")))?;
+        attrs.sort_by(|a, b| a.name.cmp(&b.name));
+        Ok((attrs, errs.iter().map(|e| e.to_string()).collect()))
+    }
+
+    /// Every link of the group whose header is `h`, sorted by name. An
+    /// object that is not a group has none.
+    pub fn links(&self, h: &ObjectHeader) -> Result<Vec<Link>> {
+        let os = self.os();
+        let ls = self.ls();
+        let data = self.data();
+        let mut out = Vec::new();
+        if let Some(m) = h
+            .messages
+            .iter()
+            .find(|m| m.msg_type == MessageType::SymbolTable)
+        {
+            let stm = SymbolTableMessage::parse(&m.data, os)
+                .map_err(|e| Error::new(format!("symbol table message: {e}")))?;
+            let entries = group_v1::resolve_v1_group_entries(data, &stm, os, ls)
+                .map_err(|e| Error::at(stm.btree_address, format!("symbol table: {e}")))?;
+            let has_soft = entries.iter().any(group_v1::is_v1_soft_link);
+            for e in entries {
+                if !group_v1::is_v1_soft_link(&e) {
+                    out.push(Link {
+                        name: e.name,
+                        kind: LinkKind::Hard(e.object_header_address),
+                    });
+                }
+            }
+            if has_soft {
+                let soft = group_v1::v1_soft_links(data, &stm, os, ls)
+                    .map_err(|e| Error::at(stm.btree_address, format!("soft links: {e}")))?;
+                for (name, target) in soft {
+                    out.push(Link {
+                        name,
+                        kind: LinkKind::Soft(target),
+                    });
+                }
+            }
+        } else {
+            let li = match h
+                .messages
+                .iter()
+                .find(|m| m.msg_type == MessageType::LinkInfo)
+            {
+                Some(m) => Some(
+                    LinkInfoMessage::parse(&m.data, os)
+                        .map_err(|e| Error::new(format!("link info message: {e}")))?,
+                ),
+                None => None,
+            };
+            for m in h
+                .messages
+                .iter()
+                .filter(|m| m.msg_type == MessageType::Link)
+            {
+                out.push(link_from_message(&m.data, os)?);
+            }
+            if let Some(li) = li
+                && let Some(fh) = li.fractal_heap_address
+            {
+                for bytes in self.dense_heap_objects(fh, li.btree_name_index_address, 5)? {
+                    out.push(link_from_message(&bytes, os)?);
+                }
+            }
+        }
+        out.sort_by(|a, b| a.name.cmp(&b.name));
+        Ok(out)
+    }
+
+    /// The objects a dense-storage B-tree (`btree`, of record type
+    /// `name_type`: 5 for links, 8 for attributes) indexes in the fractal
+    /// heap at `heap`.
+    pub fn dense_heap_objects(
+        &self,
+        heap: u64,
+        btree: Option<u64>,
+        name_type: u8,
+    ) -> Result<Vec<Vec<u8>>> {
+        self.verified_heap(heap)?;
+        let data = self.data();
+        let os = self.os();
+        let ls = self.ls();
+        let fh = FractalHeapHeader::parse(data, to_usize(heap)?, os, ls)
+            .map_err(|e| Error::at(heap, format!("fractal heap header: {e}")))?;
+        let bt = btree.ok_or_else(|| Error::at(heap, "dense storage without a name index"))?;
+        let hdr = BTreeV2Header::parse(data, to_usize(bt)?, os, ls)
+            .map_err(|e| Error::at(bt, format!("v2 B-tree header: {e}")))?;
+        let recs = collect_btree_v2_records(data, &hdr, os, ls)
+            .map_err(|e| Error::at(bt, format!("v2 B-tree: {e}")))?;
+        // Name-index records: hash(4) + heap ID; creation-order ones: order(8) + heap ID.
+        let skip = if hdr.tree_type == name_type { 4 } else { 8 };
+        let idlen = usize::from(fh.heap_id_length);
+        let mut out = Vec::with_capacity(recs.len());
+        for r in &recs {
+            let id = r
+                .data
+                .get(skip..skip + idlen)
+                .ok_or_else(|| Error::at(bt, "v2 B-tree record shorter than a heap ID"))?;
+            let obj = fh
+                .read_managed_object(data, id, os)
+                .map_err(|e| Error::at(heap, format!("fractal heap object: {e}")))?;
+            out.push(obj);
+        }
+        Ok(out)
+    }
+
+    /// Verify every block of the fractal heap at `addr` (once per heap):
+    /// the library reads heap objects without checking the blocks'
+    /// checksums, which libhdf5 does, so a damaged block would otherwise be
+    /// read as if it were sound.
+    pub fn verified_heap(&self, addr: u64) -> Result<()> {
+        if let Some(r) = self.verified_heaps.borrow().get(&addr) {
+            return r.clone().map_or(Ok(()), Err);
+        }
+        let report = crate::heap_blocks::verify(self, addr);
+        let n = report.problems.len();
+        let r = report.problems.into_iter().next().map(|mut e| {
+            if n > 1 {
+                e.msg = format!("{} (and {} more problems in this heap)", e.msg, n - 1);
+            }
+            e
+        });
+        self.verified_heaps.borrow_mut().insert(addr, r.clone());
+        r.map_or(Ok(()), Err)
+    }
+
+    /// The global heap object `idx` of the collection at `addr` (cached per
+    /// collection).
+    pub fn heap_object(&self, addr: u64, idx: u32) -> Result<Vec<u8>> {
+        let coll = {
+            let mut cache = self.heaps.borrow_mut();
+            cache
+                .entry(addr)
+                .or_insert_with(|| match usize::try_from(addr) {
+                    Ok(a) => GlobalHeapCollection::parse(self.data(), a, self.ls())
+                        .map(Rc::new)
+                        .map_err(|e| e.to_string()),
+                    Err(_) => Err("address out of range".into()),
+                })
+                .clone()
+                .map_err(|e| Error::at(addr, format!("global heap: {e}")))?
+        };
+        let idx16 = u16::try_from(idx)
+            .map_err(|_| Error::at(addr, format!("global heap object index {idx} out of range")))?;
+        coll.get_object(idx16)
+            .map(|o| o.data.clone())
+            .ok_or_else(|| Error::at(addr, format!("global heap has no object {idx}")))
+    }
+
+    /// The dataspace of the dataset at `path` with a virtual dataset's
+    /// extent resolved from its sources (as libhdf5 reports it) instead of
+    /// the stored one.
+    pub fn resolved_dataspace(&self, path: &str, h: &ObjectHeader) -> Result<Dataspace> {
+        let mut ds = self.dataspace(h)?;
+        if matches!(self.layout(h), Ok(DataLayout::Virtual { .. })) {
+            let d = self.file.dataset(path)?;
+            ds.dimensions = d.shape()?;
+            if ds.space_type == DataspaceType::Simple
+                && let Some(m) = &ds.max_dimensions
+                && m.len() != ds.dimensions.len()
+            {
+                ds.max_dimensions = None;
+            }
+        }
+        Ok(ds)
+    }
+
+    /// Read a dataset's values (raw, in file byte order) through the
+    /// `clawhdf5` facade, refusing one larger than `max_bytes`. `ds` must be
+    /// the [resolved](Self::resolved_dataspace) dataspace.
+    pub fn read_dataset(&self, path: &str, dt: &Datatype, ds: &Dataspace) -> Result<Vec<u8>> {
+        let need = byte_len(ds, dt)?;
+        if need > self.max_bytes {
+            return Err(Error::new(format!(
+                "dataset is {need} bytes, over the {} byte limit (--max-bytes)",
+                self.max_bytes
+            ))
+            .with_kind(ErrorKind::Limit));
+        }
+        let d = self.file.dataset(path)?;
+        let raw = d.read_selection(&clawhdf5::Selection::All)?;
+        if raw.len() as u64 != need {
+            return Err(Error::new(format!(
+                "read {} bytes, expected {need}",
+                raw.len()
+            )));
+        }
+        Ok(raw)
+    }
+
+    /// Walk every object reachable by hard links from the root, depth first
+    /// in name order, calling `visit` once per link (the root is visited
+    /// first with an empty link name). Each object is described once; later
+    /// hard links to it are reported with `first_path` set.
+    pub fn walk(&self, mut visit: impl FnMut(&WalkItem<'_>)) -> Result<()> {
+        self.walk_from(self.root(), "/", &mut visit)
+    }
+
+    pub fn walk_from(
+        &self,
+        start: u64,
+        start_path: &str,
+        visit: &mut dyn FnMut(&WalkItem<'_>),
+    ) -> Result<()> {
+        let mut seen: HashMap<u64, String> = HashMap::new();
+        let mut stack: Vec<(u64, String, Option<Link>, usize)> =
+            vec![(start, start_path.to_string(), None, 0)];
+        let mut count = 0usize;
+        while let Some((addr, path, link, depth)) = stack.pop() {
+            count += 1;
+            if count > MAX_OBJECTS {
+                return Err(Error::new(format!(
+                    "more than {MAX_OBJECTS} links; stopped walking"
+                )));
+            }
+            // Soft, external and user-defined links have no address.
+            if let Some(Link { kind, .. }) = &link
+                && !matches!(kind, LinkKind::Hard(_))
+            {
+                visit(&WalkItem {
+                    path: &path,
+                    link: link.as_ref(),
+                    addr: None,
+                    first_path: None,
+                    header: None,
+                    depth,
+                });
+                continue;
+            }
+            if let Some(first) = seen.get(&addr) {
+                visit(&WalkItem {
+                    path: &path,
+                    link: link.as_ref(),
+                    addr: Some(addr),
+                    first_path: Some(first),
+                    header: None,
+                    depth,
+                });
+                continue;
+            }
+            seen.insert(addr, path.clone());
+            let header = self.header(addr);
+            visit(&WalkItem {
+                path: &path,
+                link: link.as_ref(),
+                addr: Some(addr),
+                first_path: None,
+                header: Some(&header),
+                depth,
+            });
+            let Ok(h) = &header else { continue };
+            if Kind::of(h) != Kind::Group {
+                continue;
+            }
+            // Link errors are the visitor's to report (it sees the header).
+            let Ok(links) = self.links(h) else { continue };
+            let base = if path == "/" { "" } else { path.as_str() };
+            for l in links.into_iter().rev() {
+                let child = format!("{base}/{}", l.name);
+                let a = match l.kind {
+                    LinkKind::Hard(a) => a,
+                    _ => 0,
+                };
+                stack.push((a, child, Some(l), depth + 1));
+            }
+        }
+        Ok(())
+    }
+
+    /// Resolve an absolute object path to its header address, following
+    /// soft links like the library does.
+    pub fn resolve(&self, path: &str) -> Result<u64> {
+        let p = path.trim_matches('/');
+        if p.is_empty() {
+            return Ok(self.root());
+        }
+        clawhdf5_format::group_v2::resolve_path_any(self.data(), self.sb(), p)
+            .map_err(|e| Error::new(format!("{path}: {e}")))
+    }
+}
+
+/// An owned copy of a [`WalkItem`], from [`H5::walk_collect`].
+pub struct Visited {
+    pub path: String,
+    pub addr: Option<u64>,
+    pub first_path: Option<String>,
+    pub header: Option<Result<ObjectHeader>>,
+}
+
+impl H5 {
+    /// Every step of [`H5::walk`], collected, and the walk's own error.
+    pub fn walk_collect(&self) -> (Vec<Visited>, Result<()>) {
+        let mut items = Vec::new();
+        let r = self.walk(|it| {
+            items.push(Visited {
+                path: it.path.to_string(),
+                addr: it.addr,
+                first_path: it.first_path.map(str::to_string),
+                header: it.header.cloned(),
+            });
+        });
+        (items, r)
+    }
+}
+
+/// One step of [`H5::walk`].
+pub struct WalkItem<'a> {
+    pub path: &'a str,
+    /// The link that led here (`None` for the start object).
+    pub link: Option<&'a Link>,
+    /// Header address (`None` for a soft/external/user-defined link).
+    pub addr: Option<u64>,
+    /// Set when this object was already visited under another path.
+    pub first_path: Option<&'a str>,
+    /// The parsed header, for an object seen for the first time.
+    pub header: Option<&'a Result<ObjectHeader>>,
+    pub depth: usize,
+}
+
+fn link_from_message(data: &[u8], os: u8) -> Result<Link> {
+    match LinkMessage::parse(data, os) {
+        Ok(l) => Ok(Link {
+            name: l.name,
+            kind: match l.link_target {
+                LinkTarget::Hard {
+                    object_header_address,
+                } => LinkKind::Hard(object_header_address),
+                LinkTarget::Soft { target_path } => LinkKind::Soft(target_path),
+                LinkTarget::External {
+                    filename,
+                    object_path,
+                } => LinkKind::External {
+                    file: filename,
+                    path: object_path,
+                },
+            },
+        }),
+        Err(FormatError::InvalidLinkType(t)) if t >= 65 => Ok(Link {
+            name: user_defined_link_name(data).unwrap_or_else(|| "?".into()),
+            kind: LinkKind::UserDefined(t),
+        }),
+        Err(e) => Err(Error::new(format!("link message: {e}"))),
+    }
+}
+
+/// The name of a user-defined link, which `LinkMessage::parse` refuses.
+fn user_defined_link_name(d: &[u8]) -> Option<String> {
+    // version(1) flags(1) [type(1)] [corder(8)] [cset(1)] len(1|2|4|8) name
+    let flags = *d.get(1)?;
+    let mut p = 2usize;
+    if flags & 0x08 != 0 {
+        p += 1;
+    }
+    if flags & 0x04 != 0 {
+        p += 8;
+    }
+    if flags & 0x10 != 0 {
+        p += 1;
+    }
+    let w = 1usize << (flags & 0x03);
+    let mut n = 0usize;
+    for i in 0..w {
+        n |= usize::from(*d.get(p + i)?) << (8 * i);
+    }
+    p += w;
+    let name = d.get(p..p.checked_add(n)?)?;
+    Some(String::from_utf8_lossy(name).into_owned())
+}
+
+pub fn to_usize(a: u64) -> Result<usize> {
+    usize::try_from(a).map_err(|_| Error::at(a, "address out of range"))
+}
+
+/// Number of elements a dataspace holds (0 for a null dataspace).
+pub fn num_elements(ds: &Dataspace) -> Result<u64> {
+    match ds.space_type {
+        DataspaceType::Null => Ok(0),
+        DataspaceType::Scalar => Ok(1),
+        DataspaceType::Simple => ds
+            .dimensions
+            .iter()
+            .try_fold(1u64, |a, &d| a.checked_mul(d))
+            .ok_or_else(|| Error::new("dataspace element count overflows")),
+    }
+}
+
+/// Bytes needed for all of a dataspace's elements of type `dt`.
+pub fn byte_len(ds: &Dataspace, dt: &Datatype) -> Result<u64> {
+    num_elements(ds)?
+        .checked_mul(u64::from(dt.type_size()))
+        .ok_or_else(|| Error::new("dataset byte size overflows"))
+}
+
+/// Split a `FILE[/object/path]` argument the way h5ls does: the longest
+/// prefix that is an existing file is the file.
+pub fn split_file_arg(arg: &str) -> (String, Option<String>) {
+    if Path::new(arg).is_file() {
+        return (arg.to_string(), None);
+    }
+    let mut idx: Vec<usize> = arg.match_indices('/').map(|(i, _)| i).collect();
+    idx.reverse();
+    for i in idx {
+        let (f, rest) = arg.split_at(i);
+        if !f.is_empty() && Path::new(f).is_file() {
+            return (f.to_string(), Some(rest.to_string()));
+        }
+    }
+    (arg.to_string(), None)
+}

crates/clawhdf5-tools/src/heap_blocks.rs

@@ -0,0 +1,308 @@
+//! Walk every block of a fractal heap and verify it: signatures, the
+//! back-pointer to the heap header, each block's heap offset, and the
+//! checksums (always present on indirect blocks; on direct blocks when the
+//! heap header's flag says so). The library reads only the blocks an object
+//! lives in and does not verify block checksums, so `check` does it here.
+
+use std::collections::HashSet;
+
+use clawhdf5_format::checksum::jenkins_lookup3;
+use clawhdf5_format::fractal_heap::FractalHeapHeader;
+
+use crate::h5::{Error, H5};
+
+/// Heap header flag bit 1: direct blocks carry a checksum.
+const FLAG_CHECKSUM_DBLOCKS: u8 = 0x02;
+const MAX_DEPTH: u32 = 16;
+const MAX_BLOCKS: usize = 1 << 20;
+
+#[derive(Default, Debug)]
+pub struct HeapReport {
+    pub direct_blocks: usize,
+    pub indirect_blocks: usize,
+    /// Blocks whose checksum was verified.
+    pub checksums: usize,
+    pub problems: Vec<Error>,
+}
+
+struct Walk<'a> {
+    data: &'a [u8],
+    heap: u64,
+    fh: FractalHeapHeader,
+    checksum_dblocks: bool,
+    boff_bytes: usize,
+    os: usize,
+    ls: usize,
+    seen: HashSet<u64>,
+    r: HeapReport,
+}
+
+fn le(b: &[u8]) -> u64 {
+    b.iter()
+        .take(8)
+        .enumerate()
+        .fold(0u64, |a, (i, &x)| a | (u64::from(x) << (8 * i)))
+}
+
+fn undefined(v: u64, os: usize) -> bool {
+    if os >= 8 {
+        v == u64::MAX
+    } else {
+        v == (1u64 << (8 * os)) - 1
+    }
+}
+
+fn log2(v: u64) -> u32 {
+    63u32.saturating_sub(v.max(1).leading_zeros())
+}
+
+/// Verify the fractal heap whose header is at `heap`. The header itself is
+/// parsed (and its checksum verified) by the library; an error there is
+/// returned as the only problem.
+pub fn verify(h5: &H5, heap: u64) -> HeapReport {
+    let data = h5.data();
+    let Ok(off) = usize::try_from(heap) else {
+        return HeapReport {
+            problems: vec![Error::at(heap, "fractal heap address out of range")],
+            ..Default::default()
+        };
+    };
+    let fh = match FractalHeapHeader::parse(data, off, h5.os(), h5.ls()) {
+        Ok(f) => f,
+        Err(e) => {
+            return HeapReport {
+                problems: vec![Error::at(heap, format!("fractal heap header: {e}"))],
+                ..Default::default()
+            };
+        }
+    };
+    // Flags: signature(4) version(1) heap ID length(2) filter length(2) flags(1).
+    let flags = data.get(off + 9).copied().unwrap_or(0);
+    let mut w = Walk {
+        data,
+        heap,
+        checksum_dblocks: flags & FLAG_CHECKSUM_DBLOCKS != 0 && fh.filter_pipeline.is_none(),
+        boff_bytes: usize::from(fh.max_heap_size).div_ceil(8),
+        os: usize::from(h5.os()),
+        ls: usize::from(h5.ls()),
+        fh,
+        seen: HashSet::new(),
+        r: HeapReport::default(),
+    };
+    if w.fh.table_width == 0 || w.fh.starting_block_size == 0 || !w.fh.table_width.is_power_of_two()
+    {
+        w.problem(heap, "fractal heap header: invalid doubling table geometry");
+        return w.r;
+    }
+    let root = w.fh.root_block_address;
+    if !undefined(root, w.os) {
+        if w.fh.current_rows_in_root_indirect_block == 0 {
+            let size = w.fh.starting_block_size;
+            w.direct(root, size, 0);
+        } else {
+            let rows = w.fh.current_rows_in_root_indirect_block;
+            w.indirect(root, rows, 0, 0);
+        }
+    }
+    w.r
+}
+
+impl Walk<'_> {
+    fn problem(&mut self, addr: u64, msg: impl Into<String>) {
+        self.r.problems.push(Error::at(addr, msg));
+    }
+
+    fn row_size(&self, row: usize) -> Option<u64> {
+        let s = self.fh.starting_block_size;
+        if row <= 1 {
+            Some(s)
+        } else {
+            let sh = u32::try_from(row - 1).ok()?;
+            s.checked_mul(1u64.checked_shl(sh)?)
+        }
+    }
+
+    fn max_direct_rows(&self) -> usize {
+        let ratio = (self.fh.max_direct_block_size / self.fh.starting_block_size).max(1);
+        log2(ratio) as usize + 2
+    }
+
+    fn rows_for_size(&self, size: u64) -> u16 {
+        let first = log2(self.fh.starting_block_size) + log2(u64::from(self.fh.table_width));
+        (log2(size).saturating_sub(first) + 1) as u16
+    }
+
+    /// Common block prefix: signature, version, heap header address and
+    /// block offset. Returns the position after it, or `None` after
+    /// recording a problem.
+    fn prefix(&mut self, addr: u64, sig: &[u8; 4], what: &str, heap_offset: u64) -> Option<usize> {
+        if !self.seen.insert(addr) {
+            self.problem(
+                addr,
+                format!("fractal heap {what} block reached twice (cycle)"),
+            );
+            return None;
+        }
+        if self.seen.len() > MAX_BLOCKS {
+            self.problem(self.heap, "fractal heap has too many blocks; stopped");
+            return None;
+        }
+        let Ok(start) = usize::try_from(addr) else {
+            self.problem(
+                addr,
+                format!("fractal heap {what} block address out of range"),
+            );
+            return None;
+        };
+        let hdr_len = 5 + self.os + self.boff_bytes;
+        let Some(b) = start
+            .checked_add(hdr_len)
+            .and_then(|e| self.data.get(start..e))
+        else {
+            self.problem(
+                addr,
+                format!("fractal heap {what} block lies past the end of the file"),
+            );
+            return None;
+        };
+        if &b[..4] != sig {
+            self.problem(addr, format!("fractal heap {what} block: bad signature"));
+            return None;
+        }
+        if b[4] != 0 {
+            self.problem(addr, format!("fractal heap {what} block: version {}", b[4]));
+            return None;
+        }
+        let back = le(&b[5..5 + self.os]);
+        if back != self.heap {
+            self.problem(
+                addr,
+                format!(
+                    "fractal heap {what} block points at heap header {back:#x}, not {:#x}",
+                    self.heap
+                ),
+            );
+        }
+        let boff = le(&b[5 + self.os..hdr_len]);
+        if boff != heap_offset {
+            self.problem(
+                addr,
+                format!("fractal heap {what} block has heap offset {boff}, expected {heap_offset}"),
+            );
+        }
+        Some(start + hdr_len)
+    }
+
+    fn direct(&mut self, addr: u64, size: u64, heap_offset: u64) {
+        let Some(pos) = self.prefix(addr, b"FHDB", "direct", heap_offset) else {
+            return;
+        };
+        self.r.direct_blocks += 1;
+        if self.fh.filter_pipeline.is_some() {
+            return; // stored filtered: its size on disk is not the block size
+        }
+        let start = pos - (5 + self.os + self.boff_bytes);
+        let Some(end) = usize::try_from(size)
+            .ok()
+            .and_then(|s| start.checked_add(s))
+        else {
+            self.problem(addr, "fractal heap direct block size out of range");
+            return;
+        };
+        let Some(block) = self.data.get(start..end) else {
+            self.problem(
+                addr,
+                "fractal heap direct block extends past the end of the file",
+            );
+            return;
+        };
+        if self.checksum_dblocks {
+            let Some(stored) = block.get(pos - start..pos - start + 4) else {
+                self.problem(addr, "fractal heap direct block too small for its checksum");
+                return;
+            };
+            let stored = u32::from_le_bytes([stored[0], stored[1], stored[2], stored[3]]);
+            let mut copy = block.to_vec();
+            copy[pos - start..pos - start + 4].fill(0);
+            let computed = jenkins_lookup3(&copy);
+            self.r.checksums += 1;
+            if computed != stored {
+                self.problem(
+                    addr,
+                    format!(
+                        "fractal heap direct block: checksum mismatch: stored {stored:#010x}, computed {computed:#010x}"
+                    ),
+                );
+            }
+        }
+    }
+
+    fn indirect(&mut self, addr: u64, nrows: u16, heap_offset: u64, depth: u32) {
+        if depth > MAX_DEPTH {
+            self.problem(addr, "fractal heap indirect blocks nested too deeply");
+            return;
+        }
+        let Some(mut pos) = self.prefix(addr, b"FHIB", "indirect", heap_offset) else {
+            return;
+        };
+        self.r.indirect_blocks += 1;
+        let start = pos - (5 + self.os + self.boff_bytes);
+        let width = usize::from(self.fh.table_width);
+        let filtered = self.fh.filter_pipeline.is_some();
+        let direct_rows = self.max_direct_rows();
+        let mut children: Vec<(u64, bool, u64, u64)> = Vec::new(); // addr, direct, size/rows, offset
+        let mut off = heap_offset;
+        for row in 0..usize::from(nrows) {
+            let Some(rs) = self.row_size(row) else {
+                self.problem(addr, "fractal heap row size overflows");
+                return;
+            };
+            let direct = row < direct_rows;
+            for _ in 0..width {
+                let Some(b) = self.data.get(pos..pos + self.os) else {
+                    self.problem(
+                        addr,
+                        "fractal heap indirect block extends past the end of the file",
+                    );
+                    return;
+                };
+                let child = le(b);
+                pos += self.os;
+                if direct && filtered {
+                    pos += self.ls + 4;
+                }
+                if !undefined(child, self.os) {
+                    children.push((child, direct, rs, off));
+                }
+                off = off.saturating_add(rs);
+            }
+        }
+        let Some(stored) = self.data.get(pos..pos + 4) else {
+            self.problem(
+                addr,
+                "fractal heap indirect block extends past the end of the file",
+            );
+            return;
+        };
+        let stored = u32::from_le_bytes([stored[0], stored[1], stored[2], stored[3]]);
+        let computed = jenkins_lookup3(&self.data[start..pos]);
+        self.r.checksums += 1;
+        if computed != stored {
+            self.problem(
+                addr,
+                format!(
+                    "fractal heap indirect block: checksum mismatch: stored {stored:#010x}, computed {computed:#010x}"
+                ),
+            );
+            return; // its child pointers cannot be trusted
+        }
+        for (child, direct, size, off) in children {
+            if direct {
+                self.direct(child, size, off);
+            } else {
+                let rows = self.rows_for_size(size);
+                self.indirect(child, rows, off, depth + 1);
+            }
+        }
+    }
+}

crates/clawhdf5-tools/src/info.rs

@@ -0,0 +1,245 @@
+//! Dataset facts shared by `ls`, `dump`, `stat` and `check`: shape text,
+//! layout, filters and storage.
+
+use clawhdf5_format::chunked_read::{ChunkInfo, list_chunks};
+use clawhdf5_format::data_layout::DataLayout;
+use clawhdf5_format::dataspace::{Dataspace, DataspaceType};
+use clawhdf5_format::datatype::Datatype;
+use clawhdf5_format::filter_pipeline::{FilterDescription, FilterPipeline};
+use clawhdf5_format::message_type::MessageType;
+use clawhdf5_format::object_header::ObjectHeader;
+
+use crate::h5::{Error, H5, Result};
+
+/// h5ls's `{10/Inf, 20}` shape text. `always_max` prints `cur/max` for every
+/// dimension (h5ls -v).
+pub fn shape_text(ds: &Dataspace, always_max: bool) -> String {
+    match ds.space_type {
+        DataspaceType::Null => "{NULL}".into(),
+        DataspaceType::Scalar => "{SCALAR}".into(),
+        DataspaceType::Simple => {
+            let dims: Vec<String> = ds
+                .dimensions
+                .iter()
+                .enumerate()
+                .map(|(i, &d)| {
+                    let m = ds.max_dimensions.as_ref().and_then(|m| m.get(i).copied());
+                    match m {
+                        Some(u64::MAX) => format!("{d}/Inf"),
+                        Some(m) if m != d || always_max => format!("{d}/{m}"),
+                        None if always_max => format!("{d}/{d}"),
+                        _ => d.to_string(),
+                    }
+                })
+                .collect();
+            format!("{{{}}}", dims.join(", "))
+        }
+    }
+}
+
+/// h5dump's `SIMPLE { ( 3, 4 ) / ( 3, H5S_UNLIMITED ) }`.
+pub fn dataspace_ddl(ds: &Dataspace) -> String {
+    match ds.space_type {
+        DataspaceType::Null => "NULL".into(),
+        DataspaceType::Scalar => "SCALAR".into(),
+        DataspaceType::Simple => {
+            let cur: Vec<String> = ds.dimensions.iter().map(|d| d.to_string()).collect();
+            let max: Vec<String> = ds
+                .dimensions
+                .iter()
+                .enumerate()
+                .map(
+                    |(i, &d)| match ds.max_dimensions.as_ref().and_then(|m| m.get(i).copied()) {
+                        Some(u64::MAX) => "H5S_UNLIMITED".into(),
+                        Some(m) => m.to_string(),
+                        None => d.to_string(),
+                    },
+                )
+                .collect();
+            format!(
+                "SIMPLE {{ ( {} ) / ( {} ) }}",
+                cur.join(", "),
+                max.join(", ")
+            )
+        }
+    }
+}
+
+pub fn filter_name(f: &FilterDescription) -> String {
+    let known = match f.filter_id {
+        1 => "deflate",
+        2 => "shuffle",
+        3 => "fletcher32",
+        4 => "szip",
+        5 => "nbit",
+        6 => "scaleoffset",
+        307 => "bzip2",
+        32000 => "lzf",
+        32001 => "blosc",
+        32004 => "lz4",
+        32008 => "bitshuffle",
+        32013 => "zfp",
+        32015 => "zstd",
+        32026 => "blosc2",
+        _ => "",
+    };
+    if !known.is_empty() {
+        return known.into();
+    }
+    match &f.name {
+        Some(n) if !n.is_empty() => n.clone(),
+        _ => "user-defined".into(),
+    }
+}
+
+/// `deflate-1 OPT {4}` as h5ls prints a filter.
+pub fn filter_text(f: &FilterDescription) -> String {
+    let opt = if f.flags & 1 != 0 { " OPT" } else { "" };
+    let cd = if f.client_data.is_empty() {
+        String::new()
+    } else {
+        format!(
+            " {{{}}}",
+            f.client_data
+                .iter()
+                .map(|c| c.to_string())
+                .collect::<Vec<_>>()
+                .join(", ")
+        )
+    };
+    format!("{}-{}{opt}{cd}", filter_name(f), f.filter_id)
+}
+
+pub fn layout_name(l: &DataLayout) -> &'static str {
+    match l {
+        DataLayout::Compact { .. } => "compact",
+        DataLayout::Contiguous { .. } => "contiguous",
+        DataLayout::Chunked { .. } => "chunked",
+        DataLayout::Virtual { .. } => "virtual",
+    }
+}
+
+/// Chunk index kind of a chunked layout.
+pub fn chunk_index_name(l: &DataLayout) -> &'static str {
+    match l {
+        DataLayout::Chunked {
+            version,
+            chunk_index_type,
+            ..
+        } => {
+            if *version < 4 {
+                return "v1 B-tree";
+            }
+            match chunk_index_type {
+                Some(1) => "single chunk",
+                Some(2) => "implicit",
+                Some(3) => "fixed array",
+                Some(4) => "extensible array",
+                Some(5) => "v2 B-tree",
+                _ => "unknown",
+            }
+        }
+        _ => "",
+    }
+}
+
+/// Everything about one dataset that can be learned without reading its
+/// values.
+pub struct DsInfo {
+    pub dt: Result<Datatype>,
+    pub ds: Result<Dataspace>,
+    pub layout: Result<DataLayout>,
+    pub filters: Result<Option<FilterPipeline>>,
+    pub external: bool,
+}
+
+impl DsInfo {
+    pub fn read(h5: &H5, path: &str, h: &ObjectHeader) -> DsInfo {
+        DsInfo {
+            dt: h5.datatype(h),
+            ds: h5.resolved_dataspace(path, h),
+            layout: h5.layout(h),
+            filters: h5.filters(h),
+            external: h
+                .messages
+                .iter()
+                .any(|m| m.msg_type == MessageType::ExternalDataFiles),
+        }
+    }
+
+    pub fn logical_bytes(&self) -> Option<u64> {
+        let (Ok(dt), Ok(ds)) = (&self.dt, &self.ds) else {
+            return None;
+        };
+        crate::h5::byte_len(ds, dt).ok()
+    }
+}
+
+/// Every allocated chunk (empty when none are). Errors only for a corrupt
+/// chunk index.
+pub fn chunks(
+    h5: &H5,
+    layout: &DataLayout,
+    ds: &Dataspace,
+    dt: &Datatype,
+) -> Result<Vec<ChunkInfo>> {
+    let DataLayout::Chunked { btree_address, .. } = layout else {
+        return Ok(Vec::new());
+    };
+    let Some(addr) = *btree_address else {
+        return Ok(Vec::new());
+    };
+    list_chunks(
+        h5.data(),
+        layout,
+        ds,
+        dt.type_size() as usize,
+        h5.os(),
+        h5.ls(),
+    )
+    .map(|(c, _)| c)
+    .map_err(|e| {
+        Error::at(
+            addr,
+            format!("chunk index ({}): {e}", chunk_index_name(layout)),
+        )
+    })
+}
+
+/// Bytes of raw data the dataset has allocated in the file (what libhdf5's
+/// `H5Dget_storage_size` reports).
+pub fn allocated_bytes(h5: &H5, info: &DsInfo) -> Result<u64> {
+    let layout = info.layout.as_ref().map_err(Clone::clone)?;
+    Ok(match layout {
+        DataLayout::Compact { data } => data.len() as u64,
+        DataLayout::Contiguous { address, size } => {
+            if address.is_some() {
+                *size
+            } else {
+                0
+            }
+        }
+        DataLayout::Chunked { .. } => {
+            let dt = info.dt.as_ref().map_err(Clone::clone)?;
+            let ds = info.ds.as_ref().map_err(Clone::clone)?;
+            chunks(h5, layout, ds, dt)?
+                .iter()
+                .map(|c| u64::from(c.chunk_size))
+                .sum()
+        }
+        DataLayout::Virtual { .. } => 0,
+    })
+}
+
+/// Number of hard links to an object, as its header records it.
+pub fn link_count(h: &ObjectHeader) -> u64 {
+    if let Some(rc) = h.reference_count {
+        return u64::from(rc);
+    }
+    h.messages
+        .iter()
+        .find(|m| m.msg_type == MessageType::ObjectReferenceCount)
+        .and_then(|m| m.data.get(1..5))
+        .map(|b| u64::from(u32::from_le_bytes([b[0], b[1], b[2], b[3]])))
+        .unwrap_or(1)
+}

crates/clawhdf5-tools/src/lib.rs

@@ -0,0 +1,63 @@
+//! `h5rs`: HDF5 command-line tools built on clawhdf5 alone, no libhdf5.
+//!
+//! The binary's subcommands (`ls`, `dump`, `stat`, `diff`, `check`) live in
+//! the modules below; [`run`] dispatches to them. See the crate README for
+//! the command reference.
+
+pub mod check;
+pub mod cli;
+pub mod diff;
+pub mod dtype;
+pub mod dump;
+pub mod h5;
+pub mod heap_blocks;
+pub mod info;
+pub mod ls;
+pub mod stat;
+pub mod value;
+
+use cli::{Args, Out};
+
+pub const USAGE: &str = "\
+h5rs: HDF5 tools in pure Rust (clawhdf5, no libhdf5)
+
+usage: h5rs <command> [options] ...
+
+commands:
+  ls     list objects (like h5ls)
+  dump   print a file's structure and values as DDL or JSON (like h5dump)
+  stat   object, layout, filter and storage statistics (like h5stat)
+  diff   compare two files (like h5diff)
+  check  validate a file's structure and checksums
+
+Run `h5rs <command> --help` for a command's options.";
+
+/// Run `h5rs` with `argv` (without the program name). Returns the exit
+/// status.
+pub fn run(argv: Vec<String>, out: &mut Out) -> std::io::Result<i32> {
+    let mut it = argv.into_iter();
+    let Some(cmd) = it.next() else {
+        writeln!(out.e, "{USAGE}")?;
+        return Ok(2);
+    };
+    let rest: Vec<String> = it.collect();
+    match cmd.as_str() {
+        "ls" => ls::run(&mut Args::new("ls", rest), out),
+        "dump" => dump::run(&mut Args::new("dump", rest), out),
+        "stat" => stat::run(&mut Args::new("stat", rest), out),
+        "diff" => diff::run(&mut Args::new("diff", rest), out),
+        "check" => check::run(&mut Args::new("check", rest), out),
+        "-h" | "--help" | "help" => {
+            writeln!(out.o, "{USAGE}")?;
+            Ok(0)
+        }
+        "-V" | "--version" => {
+            writeln!(out.o, "h5rs {}", env!("CARGO_PKG_VERSION"))?;
+            Ok(0)
+        }
+        other => {
+            writeln!(out.e, "h5rs: unknown command {other:?}\n\n{USAGE}")?;
+            Ok(2)
+        }
+    }
+}

crates/clawhdf5-tools/src/ls.rs

@@ -0,0 +1,385 @@
+//! `h5rs ls`: list a file's objects like h5ls.
+
+use std::collections::HashMap;
+
+use clawhdf5_format::attribute::AttributeMessage;
+use clawhdf5_format::dataspace::DataspaceType;
+use clawhdf5_format::object_header::ObjectHeader;
+
+use crate::cli::{Args, Out};
+use crate::h5::{H5, Kind, Link, LinkKind, split_file_arg};
+use crate::info::{self, DsInfo};
+use crate::value::{self, Decoder};
+
+pub const USAGE: &str = "\
+usage: h5rs ls [-r] [-v] [--max-bytes N] FILE[/OBJECT]
+
+List the objects in FILE (or in the group OBJECT), one per line, like h5ls:
+name, kind, shape and, for datasets, the datatype.
+
+  -r, --recursive   list every object below, with full paths
+  -v, --verbose     also print address, link count, layout, chunking,
+                    storage, filters, datatype and attributes
+  --max-bytes N     largest attribute value decoded for -v (default 1 GiB)
+
+Exit status: 0 listed, 1 some object could not be described, 2 error.";
+
+struct Ls<'a> {
+    h5: &'a H5,
+    verbose: bool,
+    kinds: HashMap<u64, Kind>,
+    problems: usize,
+}
+
+pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result<i32> {
+    let mut recursive = false;
+    let mut verbose = false;
+    let mut max_bytes = None;
+    let mut target = None;
+    while let Some(a) = args.next() {
+        match a.as_str() {
+            "-r" | "--recursive" => recursive = true,
+            "-v" | "--verbose" => verbose = true,
+            "-rv" | "-vr" => {
+                recursive = true;
+                verbose = true;
+            }
+            "--max-bytes" => match args.number() {
+                Some(n) => max_bytes = Some(n),
+                None => return args.usage_error(out, "--max-bytes needs a number", USAGE),
+            },
+            "-h" | "--help" => {
+                writeln!(out.o, "{USAGE}")?;
+                return Ok(0);
+            }
+            s if s.starts_with('-') && s.len() > 1 => {
+                return args.usage_error(out, &format!("unknown option {a}"), USAGE);
+            }
+            _ if target.is_none() => target = Some(a),
+            _ => return args.usage_error(out, &format!("unexpected argument {a}"), USAGE),
+        }
+    }
+    let Some(target) = target else {
+        return args.usage_error(out, "missing FILE", USAGE);
+    };
+    let (file, obj) = split_file_arg(&target);
+    let mut h5 = match H5::open(std::path::Path::new(&file)) {
+        Ok(h) => h,
+        Err(e) => {
+            writeln!(out.e, "h5rs ls: {e}")?;
+            return Ok(2);
+        }
+    };
+    if let Some(m) = max_bytes {
+        h5.max_bytes = m;
+    }
+    let mut ls = Ls {
+        h5: &h5,
+        verbose,
+        kinds: HashMap::new(),
+        problems: 0,
+    };
+    let obj_path = obj.unwrap_or_else(|| "/".into());
+    let addr = match h5.resolve(&obj_path) {
+        Ok(a) => a,
+        Err(e) => {
+            writeln!(out.e, "h5rs ls: {obj_path}: not found: {e}")?;
+            return Ok(2);
+        }
+    };
+    let header = match h5.header(addr) {
+        Ok(h) => h,
+        Err(e) => {
+            writeln!(out.e, "h5rs ls: {obj_path}: {e}")?;
+            return Ok(2);
+        }
+    };
+    let is_root = obj_path.trim_matches('/').is_empty();
+    if Kind::of(&header) != Kind::Group && !is_root {
+        // h5ls names a single object by its base name, or with -r by its
+        // path as given (without the leading slash).
+        let name = if recursive {
+            obj_path.trim_start_matches('/').to_string()
+        } else {
+            obj_path.rsplit('/').next().unwrap_or(&obj_path).to_string()
+        };
+        ls.object_line(out, &name, &obj_path, addr, &Ok(header))?;
+    } else if recursive {
+        let start = if is_root { "/" } else { "" };
+        let mut io_err = None;
+        let res = h5.walk_from(addr, start, &mut |item| {
+            if io_err.is_some() || (item.link.is_none() && !is_root) {
+                return; // listing a group's contents, not the group
+            }
+            let path = if item.path.is_empty() { "/" } else { item.path };
+            let full = if is_root {
+                path.to_string()
+            } else {
+                format!("{}{path}", obj_path.trim_end_matches('/'))
+            };
+            if let Err(e) = ls.item(
+                out,
+                path,
+                &full,
+                item.link,
+                item.addr,
+                item.first_path,
+                item.header,
+            ) {
+                io_err = Some(e);
+            }
+        });
+        if let Some(e) = io_err {
+            return Err(e);
+        }
+        if let Err(e) = res {
+            writeln!(out.e, "h5rs ls: {e}")?;
+            ls.problems += 1;
+        }
+    } else {
+        match h5.links(&header) {
+            Ok(links) => {
+                for l in &links {
+                    let (a, hdr) = match l.kind {
+                        LinkKind::Hard(a) => (Some(a), Some(h5.header(a))),
+                        _ => (None, None),
+                    };
+                    let full = format!("{}/{}", obj_path.trim_end_matches('/'), l.name);
+                    ls.item(out, &l.name, &full, Some(l), a, None, hdr.as_ref())?;
+                }
+            }
+            Err(e) => {
+                writeln!(out.e, "h5rs ls: {obj_path}: cannot list group: {e}")?;
+                ls.problems += 1;
+            }
+        }
+    }
+    Ok(if ls.problems > 0 { 1 } else { 0 })
+}
+
+impl Ls<'_> {
+    #[allow(clippy::too_many_arguments)]
+    fn item(
+        &mut self,
+        out: &mut Out,
+        name: &str,
+        full: &str,
+        link: Option<&Link>,
+        addr: Option<u64>,
+        first: Option<&str>,
+        header: Option<&crate::h5::Result<ObjectHeader>>,
+    ) -> std::io::Result<()> {
+        if let Some(l) = link {
+            match &l.kind {
+                LinkKind::Soft(t) => return writeln!(out.o, "{name:<24} Soft Link {{{t}}}"),
+                LinkKind::External { file, path } => {
+                    return writeln!(out.o, "{name:<24} External Link {{{file}/{path}}}");
+                }
+                LinkKind::UserDefined(t) => {
+                    return writeln!(out.o, "{name:<24} User-defined link (type {t})");
+                }
+                LinkKind::Hard(_) => {}
+            }
+        }
+        let Some(addr) = addr else { return Ok(()) };
+        if let Some(first) = first {
+            let k = self.kinds.get(&addr).copied().unwrap_or(Kind::Unknown);
+            return writeln!(out.o, "{name:<24} {}, same as {first}", k.name());
+        }
+        match header {
+            Some(h) => self.object_line(out, name, full, addr, h),
+            None => Ok(()),
+        }
+    }
+
+    fn object_line(
+        &mut self,
+        out: &mut Out,
+        name: &str,
+        full: &str,
+        addr: u64,
+        header: &crate::h5::Result<ObjectHeader>,
+    ) -> std::io::Result<()> {
+        let h = match header {
+            Ok(h) => h,
+            Err(e) => {
+                self.problems += 1;
+                writeln!(out.o, "{name:<24} ** error **")?;
+                return writeln!(out.e, "h5rs ls: {name}: {e}");
+            }
+        };
+        let kind = Kind::of(h);
+        self.kinds.insert(addr, kind);
+        match kind {
+            Kind::Dataset => {
+                let info = DsInfo::read(self.h5, full, h);
+                let shape = match &info.ds {
+                    Ok(ds) => info::shape_text(ds, self.verbose),
+                    Err(_) => "{?}".into(),
+                };
+                if self.verbose {
+                    writeln!(out.o, "{name:<24} Dataset {shape}")?;
+                } else {
+                    let t = match &info.dt {
+                        Ok(dt) => crate::dtype::short(dt),
+                        Err(_) => "?".into(),
+                    };
+                    writeln!(out.o, "{name:<24} Dataset {shape} {t}")?;
+                }
+                for e in [
+                    info.dt.as_ref().err(),
+                    info.ds.as_ref().err(),
+                    info.layout.as_ref().err(),
+                    info.filters.as_ref().err(),
+                ]
+                .into_iter()
+                .flatten()
+                {
+                    self.problems += 1;
+                    writeln!(out.e, "h5rs ls: {name}: {e}")?;
+                }
+                if self.verbose {
+                    self.dataset_details(out, addr, h, &info)?;
+                }
+            }
+            k => {
+                writeln!(out.o, "{name:<24} {}", k.name())?;
+                if self.verbose {
+                    writeln!(out.o, "    Address:   {addr}")?;
+                    writeln!(out.o, "    Links:     {}", info::link_count(h))?;
+                    if k == Kind::Datatype
+                        && let Ok(dt) = self.h5.datatype(h)
+                    {
+                        writeln!(out.o, "    Type:      {}", crate::dtype::long(&dt))?;
+                    }
+                    self.attributes(out, name, h)?;
+                }
+            }
+        }
+        Ok(())
+    }
+
+    fn dataset_details(
+        &mut self,
+        out: &mut Out,
+        addr: u64,
+        h: &ObjectHeader,
+        info: &DsInfo,
+    ) -> std::io::Result<()> {
+        writeln!(out.o, "    Address:   {addr}")?;
+        writeln!(out.o, "    Links:     {}", info::link_count(h))?;
+        if let Ok(l) = &info.layout {
+            let idx = info::chunk_index_name(l);
+            if idx.is_empty() {
+                writeln!(out.o, "    Layout:    {}", info::layout_name(l))?;
+            } else {
+                writeln!(
+                    out.o,
+                    "    Layout:    {} ({idx} index)",
+                    info::layout_name(l)
+                )?;
+            }
+            if let clawhdf5_format::data_layout::DataLayout::Chunked {
+                chunk_dimensions, ..
+            } = l
+            {
+                let rank = chunk_dimensions.len().saturating_sub(1);
+                let dims = &chunk_dimensions[..rank];
+                let esize = info
+                    .dt
+                    .as_ref()
+                    .map(|d| u64::from(d.type_size()))
+                    .unwrap_or(0);
+                let bytes = dims
+                    .iter()
+                    .try_fold(esize, |a, &d| a.checked_mul(u64::from(d)))
+                    .map(|b| b.to_string())
+                    .unwrap_or_else(|| "?".into());
+                writeln!(
+                    out.o,
+                    "    Chunks:    {{{}}} {bytes} bytes",
+                    dims.iter()
+                        .map(|d| d.to_string())
+                        .collect::<Vec<_>>()
+                        .join(", ")
+                )?;
+            }
+        }
+        if info.external {
+            writeln!(out.o, "    External:  raw data in external files")?;
+        }
+        let logical = info.logical_bytes();
+        match (logical, info::allocated_bytes(self.h5, info)) {
+            (Some(l), Ok(a)) => {
+                if a > 0 {
+                    writeln!(
+                        out.o,
+                        "    Storage:   {l} logical bytes, {a} allocated bytes, {:.2}% utilization",
+                        l as f64 * 100.0 / a as f64
+                    )?;
+                } else {
+                    writeln!(out.o, "    Storage:   {l} logical bytes, 0 allocated bytes")?;
+                }
+            }
+            (_, Err(e)) => {
+                self.problems += 1;
+                writeln!(out.e, "h5rs ls: {e}")?;
+            }
+            _ => {}
+        }
+        if let Ok(Some(p)) = &info.filters {
+            for (i, f) in p.filters.iter().enumerate() {
+                writeln!(out.o, "    Filter-{i}:  {}", info::filter_text(f))?;
+            }
+        }
+        if let Ok(dt) = &info.dt {
+            writeln!(out.o, "    Type:      {}", crate::dtype::long(dt))?;
+        }
+        self.attributes(out, "", h)
+    }
+
+    fn attributes(&mut self, out: &mut Out, name: &str, h: &ObjectHeader) -> std::io::Result<()> {
+        let (attrs, errs) = match self.h5.attributes(h) {
+            Ok(x) => x,
+            Err(e) => {
+                self.problems += 1;
+                return writeln!(out.e, "h5rs ls: {name}: {e}");
+            }
+        };
+        for e in errs {
+            self.problems += 1;
+            writeln!(out.e, "h5rs ls: {name}: attribute: {e}")?;
+        }
+        for a in &attrs {
+            self.attribute(out, a)?;
+        }
+        Ok(())
+    }
+
+    fn attribute(&mut self, out: &mut Out, a: &AttributeMessage) -> std::io::Result<()> {
+        let shape = match a.dataspace.space_type {
+            DataspaceType::Scalar => "scalar".to_string(),
+            DataspaceType::Null => "null".to_string(),
+            DataspaceType::Simple => info::shape_text(&a.dataspace, false),
+        };
+        writeln!(out.o, "    Attribute: {} {shape}", a.name)?;
+        writeln!(
+            out.o,
+            "        Type:      {}",
+            crate::dtype::long(&a.datatype)
+        )?;
+        let n = crate::h5::num_elements(&a.dataspace).unwrap_or(0);
+        if n == 0 {
+            return Ok(());
+        }
+        let dec = Decoder::new(self.h5);
+        let shown = n.min(8) as usize;
+        let mut vals = Vec::with_capacity(shown);
+        for i in 0..shown {
+            let v = dec.element(&a.datatype, &a.raw_data, i);
+            vals.push(value::text(&v, &|_| None));
+        }
+        let more = if n > 8 { ", ..." } else { "" };
+        writeln!(out.o, "        Data:      {}{more}", vals.join(", "))
+    }
+}

crates/clawhdf5-tools/src/main.rs

@@ -0,0 +1,58 @@
+//! The `h5rs` binary. A panic anywhere is a bug: it is caught, reported as
+//! an internal error and turned into exit status 3, never a crash.
+
+use std::io::Write;
+use std::panic::{self, AssertUnwindSafe};
+
+use clawhdf5_tools::cli::Out;
+
+/// Exit status for an internal error (a caught panic).
+const INTERNAL_ERROR: i32 = 3;
+
+fn main() {
+    panic::set_hook(Box::new(|info| {
+        let msg = if let Some(s) = info.payload().downcast_ref::<&str>() {
+            (*s).to_string()
+        } else if let Some(s) = info.payload().downcast_ref::<String>() {
+            s.clone()
+        } else {
+            "unknown panic".into()
+        };
+        let at = info
+            .location()
+            .map(|l| format!(" at {}:{}", l.file(), l.line()))
+            .unwrap_or_default();
+        eprintln!("h5rs: internal error (this is a bug; please report it): {msg}{at}");
+    }));
+    let argv: Vec<String> = std::env::args().skip(1).collect();
+    let stdout = std::io::stdout();
+    let mut o = std::io::BufWriter::new(stdout.lock());
+    let mut e = std::io::stderr();
+    let result = panic::catch_unwind(AssertUnwindSafe(|| {
+        let mut out = Out {
+            o: &mut o,
+            e: &mut e,
+        };
+        clawhdf5_tools::run(argv, &mut out)
+    }));
+    let code = match result {
+        Ok(Ok(code)) => match o.flush() {
+            Ok(()) => code,
+            Err(err) if err.kind() == std::io::ErrorKind::BrokenPipe => code,
+            Err(err) => {
+                eprintln!("h5rs: {err}");
+                2
+            }
+        },
+        Ok(Err(err)) if err.kind() == std::io::ErrorKind::BrokenPipe => 0,
+        Ok(Err(err)) => {
+            eprintln!("h5rs: {err}");
+            2
+        }
+        Err(_) => {
+            let _ = o.flush();
+            INTERNAL_ERROR
+        }
+    };
+    std::process::exit(code);
+}

crates/clawhdf5-tools/src/stat.rs

@@ -0,0 +1,305 @@
+//! `h5rs stat`: object, layout, filter and storage statistics, laid out like
+//! h5stat's report.
+
+use std::collections::BTreeMap;
+
+use clawhdf5_format::data_layout::DataLayout;
+use clawhdf5_format::dataspace::DataspaceType;
+
+use crate::cli::{Args, Out};
+use crate::h5::{H5, Kind, LinkKind};
+use crate::info::{self, DsInfo};
+
+pub const USAGE: &str = "\
+usage: h5rs stat FILE
+
+Print statistics for FILE, in the layout of h5stat's report: object counts,
+links, dataset ranks, layouts, filters, attribute counts, raw-data size and
+a file-space summary. Metadata space is not broken down by structure (h5stat
+does); it is reported as one figure together with free space.
+
+Exit status: 0 printed, 1 some object could not be read (counted as
+\"unreadable\"), 2 error.";
+
+#[derive(Default)]
+struct Stats {
+    groups: u64,
+    datasets: u64,
+    datatypes: u64,
+    other: u64,
+    unreadable: u64,
+    links: u64,
+    max_links_to_object: u64,
+    max_objects_in_group: u64,
+    group_sizes: BTreeMap<usize, u64>,
+    ranks: BTreeMap<usize, u64>,
+    max_1d: u64,
+    layout: [u64; 4],
+    external: u64,
+    no_filter: u64,
+    filter_counts: BTreeMap<&'static str, u64>,
+    raw_data: u64,
+    raw_errors: u64,
+    attr_objects: u64,
+    max_attrs: u64,
+    attr_counts: BTreeMap<usize, u64>,
+}
+
+const FILTER_KINDS: [&str; 8] = [
+    "GZIP",
+    "SHUFFLE",
+    "FLETCHER32",
+    "SZIP",
+    "NBIT",
+    "SCALEOFFSET",
+    "USER-DEFINED",
+    "",
+];
+
+fn filter_kind(id: u16) -> &'static str {
+    match id {
+        1 => "GZIP",
+        2 => "SHUFFLE",
+        3 => "FLETCHER32",
+        4 => "SZIP",
+        5 => "NBIT",
+        6 => "SCALEOFFSET",
+        _ => "USER-DEFINED",
+    }
+}
+
+pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result<i32> {
+    let mut file = None;
+    while let Some(a) = args.next() {
+        match a.as_str() {
+            "-h" | "--help" => {
+                writeln!(out.o, "{USAGE}")?;
+                return Ok(0);
+            }
+            s if s.starts_with('-') && s.len() > 1 => {
+                return args.usage_error(out, &format!("unknown option {a}"), USAGE);
+            }
+            _ if file.is_none() => file = Some(a),
+            _ => return args.usage_error(out, &format!("unexpected argument {a}"), USAGE),
+        }
+    }
+    let Some(file) = file else {
+        return args.usage_error(out, "missing FILE", USAGE);
+    };
+    let h5 = match H5::open(std::path::Path::new(&file)) {
+        Ok(h) => h,
+        Err(e) => {
+            writeln!(out.e, "h5rs stat: {e}")?;
+            return Ok(2);
+        }
+    };
+    let mut s = Stats::default();
+    let mut errors: Vec<String> = Vec::new();
+    let walk = h5.walk(|it| {
+        if let Some(l) = it.link
+            && !matches!(l.kind, LinkKind::Hard(_))
+        {
+            s.links += 1;
+            return;
+        }
+        if it.first_path.is_some() {
+            return;
+        }
+        let Some(Ok(h)) = it.header else {
+            if let Some(Err(e)) = it.header {
+                s.unreadable += 1;
+                errors.push(format!("{}: {e}", it.path));
+            }
+            return;
+        };
+        s.max_links_to_object = s.max_links_to_object.max(info::link_count(h));
+        match h5.attributes(h) {
+            Ok((a, errs)) => {
+                let n = a.len() + errs.len();
+                if n > 0 {
+                    s.attr_objects += 1;
+                    s.max_attrs = s.max_attrs.max(n as u64);
+                    *s.attr_counts.entry(n).or_default() += 1;
+                }
+                for e in errs {
+                    errors.push(format!("{}: attribute: {e}", it.path));
+                }
+            }
+            Err(e) => errors.push(format!("{}: {e}", it.path)),
+        }
+        match Kind::of(h) {
+            Kind::Group => {
+                s.groups += 1;
+                match h5.links(h) {
+                    Ok(l) => {
+                        s.max_objects_in_group = s.max_objects_in_group.max(l.len() as u64);
+                        *s.group_sizes.entry(l.len()).or_default() += 1;
+                    }
+                    Err(e) => {
+                        s.unreadable += 1;
+                        errors.push(format!("{}: {e}", it.path));
+                    }
+                }
+            }
+            Kind::Datatype => s.datatypes += 1,
+            Kind::Unknown => {
+                // The root group of a file with no links at all has no
+                // group messages; count it as a group.
+                if it.link.is_none() {
+                    s.groups += 1;
+                    *s.group_sizes.entry(0).or_default() += 1;
+                } else {
+                    s.other += 1;
+                }
+            }
+            Kind::Dataset => {
+                s.datasets += 1;
+                let info = DsInfo::read(&h5, it.path, h);
+                if let Ok(ds) = &info.ds {
+                    let rank = match ds.space_type {
+                        DataspaceType::Simple => ds.dimensions.len(),
+                        _ => 0,
+                    };
+                    *s.ranks.entry(rank).or_default() += 1;
+                    if rank == 1 {
+                        s.max_1d = s.max_1d.max(ds.dimensions[0]);
+                    }
+                }
+                match &info.layout {
+                    Ok(l) => {
+                        let i = match l {
+                            DataLayout::Compact { .. } => 0,
+                            DataLayout::Contiguous { .. } => 1,
+                            DataLayout::Chunked { .. } => 2,
+                            DataLayout::Virtual { .. } => 3,
+                        };
+                        s.layout[i] += 1;
+                    }
+                    Err(e) => errors.push(format!("{}: {e}", it.path)),
+                }
+                if info.external {
+                    s.external += 1;
+                }
+                match &info.filters {
+                    Ok(Some(p)) if !p.filters.is_empty() => {
+                        let mut kinds: Vec<&'static str> =
+                            p.filters.iter().map(|f| filter_kind(f.filter_id)).collect();
+                        kinds.sort_unstable();
+                        kinds.dedup();
+                        for k in kinds {
+                            *s.filter_counts.entry(k).or_default() += 1;
+                        }
+                    }
+                    Ok(_) => s.no_filter += 1,
+                    Err(e) => errors.push(format!("{}: {e}", it.path)),
+                }
+                match info::allocated_bytes(&h5, &info) {
+                    Ok(b) => s.raw_data = s.raw_data.saturating_add(b),
+                    Err(e) => {
+                        s.raw_errors += 1;
+                        errors.push(format!("{}: storage size: {e}", it.path));
+                    }
+                }
+            }
+        }
+    });
+    if let Err(e) = walk {
+        errors.push(e.to_string());
+    }
+    report(&h5, &file, &s, out)?;
+    for e in &errors {
+        writeln!(out.e, "h5rs stat: {e}")?;
+    }
+    Ok(if errors.is_empty() { 0 } else { 1 })
+}
+
+fn report(h5: &H5, file: &str, s: &Stats, out: &mut Out) -> std::io::Result<()> {
+    let o = &mut out.o;
+    writeln!(o, "Filename: {file}")?;
+    writeln!(o, "File information")?;
+    writeln!(o, "\t# of unique groups: {}", s.groups)?;
+    writeln!(o, "\t# of unique datasets: {}", s.datasets)?;
+    writeln!(o, "\t# of unique named datatypes: {}", s.datatypes)?;
+    writeln!(o, "\t# of unique links: {}", s.links)?;
+    writeln!(o, "\t# of unique other: {}", s.other)?;
+    if s.unreadable > 0 {
+        writeln!(o, "\t# of unreadable objects: {}", s.unreadable)?;
+    }
+    writeln!(o, "\tMax. # of links to object: {}", s.max_links_to_object)?;
+    writeln!(
+        o,
+        "\tMax. # of objects in group: {}",
+        s.max_objects_in_group
+    )?;
+    let sb = h5.sb();
+    writeln!(o, "Superblock:")?;
+    writeln!(o, "\tVersion: {}", sb.version)?;
+    writeln!(o, "\tSize of offsets: {} bytes", sb.offset_size)?;
+    writeln!(o, "\tSize of lengths: {} bytes", sb.length_size)?;
+    writeln!(o, "\tUser block: {} bytes", h5.file.user_block_size())?;
+    writeln!(o, "Group bins:")?;
+    for (n, c) in &s.group_sizes {
+        writeln!(o, "\t# of groups with {n} link(s): {c}")?;
+    }
+    writeln!(o, "\tTotal # of groups: {}", s.groups)?;
+    writeln!(o, "Dataset dimension information:")?;
+    writeln!(
+        o,
+        "\tMax. rank of datasets: {}",
+        s.ranks.keys().next_back().copied().unwrap_or(0)
+    )?;
+    writeln!(o, "\tDataset ranks:")?;
+    for (r, c) in &s.ranks {
+        writeln!(o, "\t\t# of dataset with rank {r}: {c}")?;
+    }
+    writeln!(o, "1-D Dataset information:")?;
+    writeln!(o, "\tMax. dimension size of 1-D datasets: {}", s.max_1d)?;
+    writeln!(o, "Dataset storage information:")?;
+    writeln!(o, "\tTotal raw data size: {}", s.raw_data)?;
+    if s.raw_errors > 0 {
+        writeln!(
+            o,
+            "\tDatasets whose storage size could not be read: {}",
+            s.raw_errors
+        )?;
+    }
+    writeln!(o, "Dataset layout information:")?;
+    for (i, n) in ["COMPACT", "CONTIG", "CHUNKED", "VIRTUAL"]
+        .iter()
+        .enumerate()
+    {
+        writeln!(o, "\tDataset layout counts[{n}]: {}", s.layout[i])?;
+    }
+    writeln!(o, "\tDatasets with external raw data: {}", s.external)?;
+    writeln!(o, "Dataset filters information:")?;
+    writeln!(o, "\tNumber of datasets with:")?;
+    writeln!(o, "\t\tNO filter: {}", s.no_filter)?;
+    for k in FILTER_KINDS.iter().filter(|k| !k.is_empty()) {
+        writeln!(
+            o,
+            "\t\t{k} filter: {}",
+            s.filter_counts.get(k).copied().unwrap_or(0)
+        )?;
+    }
+    writeln!(o, "Attribute information:")?;
+    for (n, c) in &s.attr_counts {
+        writeln!(o, "\t# of objects with {n} attribute(s): {c}")?;
+    }
+    writeln!(
+        o,
+        "\tTotal # of objects with attributes: {}",
+        s.attr_objects
+    )?;
+    writeln!(o, "\tMax. # of attributes to objects: {}", s.max_attrs)?;
+    let total = std::fs::metadata(&h5.path).map(|m| m.len()).unwrap_or(0);
+    let ub = h5.file.user_block_size();
+    writeln!(o, "Summary of file space information:")?;
+    writeln!(o, "  User block: {ub} bytes")?;
+    writeln!(o, "  Raw data: {} bytes", s.raw_data)?;
+    writeln!(
+        o,
+        "  Metadata and free space: {} bytes",
+        total.saturating_sub(s.raw_data).saturating_sub(ub)
+    )?;
+    writeln!(o, "Total space: {total} bytes")
+}

crates/clawhdf5-tools/src/value.rs

@@ -0,0 +1,483 @@
+//! Decoding one element of any datatype into a [`Value`], and printing it.
+//!
+//! Decoding never panics: a short buffer, an unknown byte order or a
+//! dangling heap reference becomes [`Value::Error`].
+
+use clawhdf5_format::datatype::{Datatype, DatatypeByteOrder, ReferenceType, StringPadding};
+use serde_json::Value as J;
+
+use crate::dtype;
+use crate::h5::H5;
+
+#[derive(Debug, Clone, PartialEq)]
+pub enum Value {
+    Int(i128),
+    /// A float and the width (in bits) it was stored with, so it prints at
+    /// its own precision.
+    Float(f64, u8),
+    Str(String),
+    /// Opaque, bitfield, time and oversized integers.
+    Bytes(Vec<u8>),
+    /// An enum member (name, when the value matches one) and its value.
+    Enum(Option<String>, i128),
+    Compound(Vec<(String, Value)>),
+    Array(Vec<Value>),
+    /// A variable-length sequence.
+    Seq(Vec<Value>),
+    /// A reference: the referenced object's address (`None` = null).
+    Ref(Option<u64>),
+    /// A region or attribute reference, kept as its bytes.
+    OtherRef(Vec<u8>),
+    Error(String),
+}
+
+pub fn hex(b: &[u8]) -> String {
+    let mut s = String::from("0x");
+    for x in b {
+        s.push_str(&format!("{x:02x}"));
+    }
+    s
+}
+
+/// Element bytes as an unsigned integer (at most 16 bytes).
+fn bits(b: &[u8], order: &DatatypeByteOrder) -> Option<u128> {
+    if b.len() > 16 {
+        return None;
+    }
+    let mut v = 0u128;
+    match order {
+        DatatypeByteOrder::LittleEndian => {
+            for (i, x) in b.iter().enumerate() {
+                v |= u128::from(*x) << (8 * i);
+            }
+        }
+        DatatypeByteOrder::BigEndian => {
+            for x in b {
+                v = (v << 8) | u128::from(*x);
+            }
+        }
+        DatatypeByteOrder::Vax => return None,
+    }
+    Some(v)
+}
+
+/// The value of an integer (fixed-point) element, honouring its bit offset
+/// and precision. `None` when it cannot be represented (over 16 bytes) or
+/// `dt` is not an integer.
+pub fn decode_int(dt: &Datatype, b: &[u8]) -> Option<i128> {
+    let Datatype::FixedPoint {
+        size,
+        byte_order,
+        signed,
+        bit_offset,
+        bit_precision,
+    } = dt
+    else {
+        return None;
+    };
+    let size = usize::try_from(*size).ok()?;
+    let v = bits(b.get(..size)?, byte_order)?;
+    let off = u32::from(*bit_offset);
+    let prec = u32::from(*bit_precision).min(128);
+    if off >= 128 || prec == 0 {
+        return Some(0);
+    }
+    let mut x = v >> off;
+    if prec < 128 {
+        x &= (1u128 << prec) - 1;
+    }
+    if *signed && prec < 128 && (x >> (prec - 1)) & 1 == 1 {
+        x |= !0u128 << prec;
+        return Some(x as i128);
+    }
+    if !*signed && prec == 128 && x > i128::MAX as u128 {
+        return None;
+    }
+    Some(x as i128)
+}
+
+fn decode_float(dt: &Datatype, b: &[u8]) -> Value {
+    let Datatype::FloatingPoint {
+        size, byte_order, ..
+    } = dt
+    else {
+        return Value::Error("not a float".into());
+    };
+    let Ok(n) = usize::try_from(*size) else {
+        return Value::Error("float size".into());
+    };
+    let Some(b) = b.get(..n) else {
+        return Value::Error("short element".into());
+    };
+    if dtype::is_ieee(dt)
+        && let Some(v) = bits(b, byte_order)
+    {
+        return match n {
+            2 => Value::Float(
+                f64::from(clawhdf5_format::float16::f16_bits_to_f32(v as u16)),
+                16,
+            ),
+            4 => Value::Float(f64::from(f32::from_bits(v as u32)), 32),
+            _ => Value::Float(f64::from_bits(v as u64), 64),
+        };
+    }
+    // Non-IEEE layouts (N-Bit floats, VAX order): the library converts.
+    match clawhdf5_format::data_read::read_as_f64(b, dt) {
+        Ok(v) if v.len() == 1 => Value::Float(v[0], (n * 8).min(64) as u8),
+        Ok(_) => Value::Error("float conversion".into()),
+        Err(e) => Value::Error(e.to_string()),
+    }
+}
+
+fn trim_string(b: &[u8], pad: Option<&StringPadding>) -> String {
+    let cut = b.iter().position(|&c| c == 0).unwrap_or(b.len());
+    let mut s = &b[..cut];
+    if matches!(pad, Some(StringPadding::SpacePad)) {
+        while let [rest @ .., b' '] = s {
+            s = rest;
+        }
+    }
+    String::from_utf8_lossy(s).into_owned()
+}
+
+/// Little-endian unsigned integer of `b` (up to 8 bytes).
+fn le(b: &[u8]) -> u64 {
+    b.iter()
+        .take(8)
+        .enumerate()
+        .fold(0u64, |a, (i, &x)| a | (u64::from(x) << (8 * i)))
+}
+
+/// Decodes elements of one file.
+pub struct Decoder<'a> {
+    pub h5: &'a H5,
+}
+
+impl<'a> Decoder<'a> {
+    pub fn new(h5: &'a H5) -> Self {
+        Self { h5 }
+    }
+
+    /// Decode element `i` of `raw`, an array of `dt` elements.
+    pub fn element(&self, dt: &Datatype, raw: &[u8], i: usize) -> Value {
+        let size = dt.type_size() as usize;
+        match i
+            .checked_mul(size)
+            .and_then(|s| raw.get(s..s.checked_add(size)?))
+        {
+            Some(b) => self.decode(dt, b, 0),
+            None => Value::Error("element out of range".into()),
+        }
+    }
+
+    pub fn decode(&self, dt: &Datatype, b: &[u8], depth: u32) -> Value {
+        if depth > 32 {
+            return Value::Error("datatype nesting too deep".into());
+        }
+        let size = dt.type_size() as usize;
+        let Some(b) = b.get(..size) else {
+            return Value::Error("short element".into());
+        };
+        match dt {
+            Datatype::FixedPoint { .. } => match decode_int(dt, b) {
+                Some(v) => Value::Int(v),
+                None => Value::Bytes(b.to_vec()),
+            },
+            Datatype::FloatingPoint { .. } => decode_float(dt, b),
+            Datatype::Time { .. } | Datatype::BitField { .. } | Datatype::Opaque { .. } => {
+                Value::Bytes(b.to_vec())
+            }
+            Datatype::String { padding, .. } => Value::Str(trim_string(b, Some(padding))),
+            Datatype::Compound { members, .. } => {
+                let mut out = Vec::with_capacity(members.len());
+                for m in members {
+                    let off = usize::try_from(m.byte_offset).unwrap_or(usize::MAX);
+                    let v = match b.get(off..) {
+                        Some(mb) => self.decode(&m.datatype, mb, depth + 1),
+                        None => Value::Error("member out of bounds".into()),
+                    };
+                    out.push((m.name.clone(), v));
+                }
+                Value::Compound(out)
+            }
+            Datatype::Reference { ref_type, .. } => match ref_type {
+                ReferenceType::Object | ReferenceType::Object2 => {
+                    match clawhdf5_format::data_read::read_object_references(b, dt, self.h5.os()) {
+                        Ok(r) if r.len() == 1 => {
+                            let a = r[0].address;
+                            let undef = a == u64::MAX
+                                || (self.h5.os() < 8 && a == (1u64 << (8 * self.h5.os())) - 1);
+                            Value::Ref(if undef || a == 0 { None } else { Some(a) })
+                        }
+                        Ok(_) => Value::Error("reference".into()),
+                        Err(e) => Value::Error(e.to_string()),
+                    }
+                }
+                _ => Value::OtherRef(b.to_vec()),
+            },
+            Datatype::Enumeration {
+                base_type, members, ..
+            } => {
+                let Some(v) = decode_int(base_type, b) else {
+                    return Value::Bytes(b.to_vec());
+                };
+                let bs = base_type.type_size() as usize;
+                let name = members
+                    .iter()
+                    .find(|m| m.value.get(..bs) == b.get(..bs))
+                    .map(|m| m.name.clone());
+                Value::Enum(name, v)
+            }
+            Datatype::Array {
+                base_type,
+                dimensions,
+            } => {
+                let n = dimensions
+                    .iter()
+                    .try_fold(1usize, |a, &d| a.checked_mul(d as usize));
+                let bs = base_type.type_size() as usize;
+                let Some(n) = n.filter(|n| n.checked_mul(bs).is_some_and(|t| t <= b.len())) else {
+                    return Value::Error("array larger than its element".into());
+                };
+                let mut out = Vec::with_capacity(n);
+                for k in 0..n {
+                    out.push(self.decode(base_type, &b[k * bs..], depth + 1));
+                }
+                Value::Array(out)
+            }
+            Datatype::VariableLength {
+                is_string,
+                padding,
+                base_type,
+                ..
+            } => self.decode_vlen(*is_string, padding.as_ref(), base_type, b, depth),
+        }
+    }
+
+    fn decode_vlen(
+        &self,
+        is_string: bool,
+        padding: Option<&StringPadding>,
+        base: &Datatype,
+        b: &[u8],
+        depth: u32,
+    ) -> Value {
+        let os = usize::from(self.h5.os());
+        let (Some(lenb), Some(addrb), Some(idxb)) =
+            (b.get(..4), b.get(4..4 + os), b.get(4 + os..8 + os))
+        else {
+            return Value::Error("short VL element".into());
+        };
+        let len = le(lenb) as usize;
+        let addr = le(addrb);
+        let idx = le(idxb) as u32;
+        let undef = if os >= 8 {
+            u64::MAX
+        } else {
+            (1u64 << (8 * os)) - 1
+        };
+        let obj = if len == 0 || addr == 0 || addr == undef {
+            Vec::new()
+        } else {
+            match self.h5.heap_object(addr, idx) {
+                Ok(o) => o,
+                Err(e) => return Value::Error(e.to_string()),
+            }
+        };
+        if is_string {
+            let l = len.min(obj.len());
+            return Value::Str(trim_string(&obj[..l], padding));
+        }
+        let bs = base.type_size() as usize;
+        if bs == 0 {
+            return Value::Error("VL base type of size 0".into());
+        }
+        match len.checked_mul(bs) {
+            Some(need) if need <= obj.len() => {}
+            _ => return Value::Error("VL sequence longer than its heap object".into()),
+        }
+        let mut out = Vec::with_capacity(len);
+        for k in 0..len {
+            out.push(self.decode(base, &obj[k * bs..], depth + 1));
+        }
+        Value::Seq(out)
+    }
+}
+
+/// Format a float like C's `%g` would at full round-trip precision: plain
+/// digits for moderate magnitudes, an exponent otherwise.
+pub fn fmt_float(v: f64, width: u8) -> String {
+    if v.is_nan() {
+        return "NaN".into();
+    }
+    if v.is_infinite() {
+        return if v > 0.0 { "Inf".into() } else { "-Inf".into() };
+    }
+    let a = v.abs();
+    let plain = a == 0.0 || (1e-5..1e16).contains(&a);
+    match (width, plain) {
+        (16 | 32, true) => format!("{}", v as f32),
+        (16 | 32, false) => format!("{:e}", v as f32),
+        (_, true) => format!("{v}"),
+        (_, false) => format!("{v:e}"),
+    }
+}
+
+fn escape(s: &str) -> String {
+    let mut o = String::with_capacity(s.len() + 2);
+    for c in s.chars() {
+        match c {
+            '"' => o.push_str("\\\""),
+            '\\' => o.push_str("\\\\"),
+            '\n' => o.push_str("\\n"),
+            '\r' => o.push_str("\\r"),
+            '\t' => o.push_str("\\t"),
+            c if (c as u32) < 0x20 => o.push_str(&format!("\\{:03o}", c as u32)),
+            c => o.push(c),
+        }
+    }
+    o
+}
+
+/// A fixed-length string element's bytes quoted as h5dump prints a
+/// null-padded string: every byte, NULs as `\000`.
+pub fn quote_bytes(b: &[u8]) -> String {
+    format!("\"{}\"", escape(&String::from_utf8_lossy(b)))
+}
+
+/// Text form, as in an h5dump DATA block.
+pub fn text(v: &Value, h5paths: &dyn Fn(u64) -> Option<String>) -> String {
+    match v {
+        Value::Int(i) => i.to_string(),
+        Value::Float(f, w) => fmt_float(*f, *w),
+        Value::Str(s) => format!("\"{}\"", escape(s)),
+        Value::Bytes(b) => hex(b),
+        Value::Enum(Some(n), _) => n.clone(),
+        Value::Enum(None, i) => i.to_string(),
+        Value::Compound(ms) => format!(
+            "{{ {} }}",
+            ms.iter()
+                .map(|(_, v)| text(v, h5paths))
+                .collect::<Vec<_>>()
+                .join(", ")
+        ),
+        Value::Array(vs) => format!(
+            "[ {} ]",
+            vs.iter()
+                .map(|v| text(v, h5paths))
+                .collect::<Vec<_>>()
+                .join(", ")
+        ),
+        Value::Seq(vs) => format!(
+            "({})",
+            vs.iter()
+                .map(|v| text(v, h5paths))
+                .collect::<Vec<_>>()
+                .join(", ")
+        ),
+        Value::Ref(None) => "NULL".into(),
+        Value::Ref(Some(a)) => match h5paths(*a) {
+            Some(p) => format!("\"{p}\""),
+            None => format!("{a:#x}"),
+        },
+        Value::OtherRef(b) => hex(b),
+        Value::Error(e) => format!("<error: {e}>"),
+    }
+}
+
+/// hdf5-json value form.
+pub fn to_json(v: &Value, h5paths: &dyn Fn(u64) -> Option<String>) -> J {
+    match v {
+        Value::Int(i) => {
+            if let Ok(x) = i64::try_from(*i) {
+                J::from(x)
+            } else if let Ok(x) = u64::try_from(*i) {
+                J::from(x)
+            } else {
+                J::from(i.to_string())
+            }
+        }
+        Value::Float(f, _) => {
+            if f.is_finite() {
+                serde_json::Number::from_f64(*f)
+                    .map(J::Number)
+                    .unwrap_or(J::Null)
+            } else if f.is_nan() {
+                J::from("NaN")
+            } else if *f > 0.0 {
+                J::from("Infinity")
+            } else {
+                J::from("-Infinity")
+            }
+        }
+        Value::Str(s) => J::from(s.as_str()),
+        Value::Bytes(b) | Value::OtherRef(b) => J::from(hex(b)),
+        Value::Enum(_, i) => to_json(&Value::Int(*i), h5paths),
+        Value::Compound(ms) => J::Array(ms.iter().map(|(_, v)| to_json(v, h5paths)).collect()),
+        Value::Array(vs) | Value::Seq(vs) => {
+            J::Array(vs.iter().map(|v| to_json(v, h5paths)).collect())
+        }
+        Value::Ref(None) => J::Null,
+        Value::Ref(Some(a)) => J::from(h5paths(*a).unwrap_or_else(|| format!("{a:#x}"))),
+        Value::Error(e) => serde_json::json!({ "error": e }),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn int(size: u32, signed: bool, order: DatatypeByteOrder, off: u16, prec: u16) -> Datatype {
+        Datatype::FixedPoint {
+            size,
+            byte_order: order,
+            signed,
+            bit_offset: off,
+            bit_precision: prec,
+        }
+    }
+
+    #[test]
+    fn integers_decode_with_order_offset_and_sign() {
+        let le = DatatypeByteOrder::LittleEndian;
+        let be = DatatypeByteOrder::BigEndian;
+        assert_eq!(
+            decode_int(&int(2, true, le.clone(), 0, 16), &[0xff, 0xff]),
+            Some(-1)
+        );
+        assert_eq!(
+            decode_int(&int(2, false, le, 0, 16), &[0xff, 0xff]),
+            Some(65535)
+        );
+        assert_eq!(
+            decode_int(&int(2, true, be.clone(), 0, 16), &[0x80, 0x00]),
+            Some(-32768)
+        );
+        // 17-bit signed field at offset 4: -5
+        let stored = (((-5i32) as u32) & 0x1_FFFF) << 4;
+        assert_eq!(
+            decode_int(&int(4, true, be, 4, 17), &stored.to_be_bytes()),
+            Some(-5)
+        );
+        assert_eq!(
+            decode_int(&int(4, true, DatatypeByteOrder::Vax, 0, 32), &[0; 4]),
+            None
+        );
+        assert_eq!(
+            decode_int(
+                &int(4, true, DatatypeByteOrder::LittleEndian, 0, 32),
+                &[0; 2]
+            ),
+            None
+        );
+    }
+
+    #[test]
+    fn floats_print_at_their_own_precision() {
+        assert_eq!(fmt_float(f64::from(0.1f32), 32), "0.1");
+        assert_eq!(fmt_float(0.1, 64), "0.1");
+        assert_eq!(fmt_float(1e20, 64), "1e20");
+        assert_eq!(fmt_float(2.0, 64), "2");
+        assert_eq!(fmt_float(f64::NAN, 64), "NaN");
+    }
+}

crates/clawhdf5-tools/tests/gen_files.py

@@ -0,0 +1,164 @@
+"""Write the HDF5 files the h5rs interop tests run on.
+
+usage: gen_files.py OUTDIR
+
+Writes OUTDIR/{earliest,latest}.h5 (the same content with the oldest and the
+newest file-format structures: symbol tables and v1 B-trees vs. v2 object
+headers, fractal heaps, v2 B-trees and the chunk indexes of HDF5 1.10+), the
+pairs the diff tests compare, and a file with a user block. Prints one JSON
+object with the values h5py reads back, for the dump tests.
+"""
+
+import json
+import os
+import sys
+
+import h5py
+import numpy as np
+
+out = sys.argv[1]
+
+
+def content(f, dense):
+    f.attrs["title"] = "h5rs test"
+    f.attrs["version"] = np.int64(3)
+    f.attrs["scale"] = np.array([0.5, 1.5], dtype="f4")
+    f["contig"] = np.arange(12, dtype="f8").reshape(3, 4)
+    dcpl = h5py.h5p.create(h5py.h5p.DATASET_CREATE)
+    dcpl.set_layout(h5py.h5d.COMPACT)
+    space = h5py.h5s.create_simple((4,))
+    h5py.h5d.create(f.id, b"compact", h5py.h5t.STD_I16LE, space, dcpl).write(
+        h5py.h5s.ALL, h5py.h5s.ALL, np.arange(4, dtype="<i2")
+    )
+    g = f.create_group("grp")
+    g.attrs["units"] = "m"
+    g.create_dataset(
+        "gz", data=np.arange(1000, dtype="i4"), chunks=(100,), compression="gzip",
+        shuffle=True, fletcher32=True,
+    )
+    g.create_dataset("ext1", data=np.arange(50, dtype="u2"), chunks=(8,), maxshape=(None,))
+    g.create_dataset(
+        "ext2", data=np.arange(60, dtype="f4").reshape(6, 10), chunks=(4, 4), maxshape=(None, None)
+    )
+    g.create_dataset("fixed", data=np.arange(64, dtype="i8").reshape(8, 8), chunks=(3, 3))
+    g.create_dataset("single", data=np.arange(10, dtype="i4"), chunks=(10,))
+    g.create_dataset("sparse", shape=(100,), dtype="i4", chunks=(10,), fillvalue=-1)
+    g["sparse"][20:30] = 7
+    sub = g.create_group("sub")
+    sub["scalar"] = np.float32(2.5)
+    sub["empty"] = h5py.Empty("f8")
+    f["strings"] = np.array([b"ab", b"cde"])
+    f["vlstr"] = np.array(["x", "yy", "zzz"], dtype=h5py.string_dtype())
+    f["cmp"] = np.array([(1, 2.5), (3, 4.5)], dtype=[("a", "i2"), ("b", ">f4")])
+    f.create_dataset(
+        "enum", data=np.array([0, 1, 1], dtype="u1"),
+        dtype=h5py.enum_dtype({"RED": 0, "GREEN": 1}, basetype="u1"),
+    )
+    f["be"] = np.arange(5, dtype=">i4")
+    f["arr"] = np.array([([1, 2, 3],), ([4, 5, 6],)], dtype=[("v", "3i4")])
+    f["named_t"] = np.dtype("i8")
+    f["soft"] = h5py.SoftLink("/contig")
+    f["dangling"] = h5py.SoftLink("/nowhere")
+    f["external"] = h5py.ExternalLink("other.h5", "/x")
+    f["hard2"] = g["sub"]
+    many = f.create_group("many")
+    for i in range(12 if dense else 4):
+        many[f"d{i:02}"] = np.int32(i)
+        many.attrs[f"a{i:02}"] = i
+
+
+values = {}
+# "latest" under HDF5 2.0 writes datatype messages that libhdf5 1.14 tools
+# cannot read, so the newest format is taken as 1.14's.
+for libver in ("earliest", "latest"):
+    path = os.path.join(out, f"{libver}.h5")
+    bounds = ("earliest", "v114") if libver == "earliest" else ("v114", "v114")
+    with h5py.File(path, "w", libver=bounds) as f:
+        content(f, True)
+    with h5py.File(path, "r") as f:
+        vals = {}
+
+        def grab(name, obj):
+            if isinstance(obj, h5py.Dataset) and obj.dtype.kind in "iuf" and obj.shape is not None:
+                vals["/" + name] = obj[()].tolist()
+
+        f.visititems(grab)
+        values[libver] = vals
+
+# diff pairs
+def small(path, data=None, extra=False, attr=False, shape=(3, 4), dtype="f8"):
+    with h5py.File(os.path.join(out, path), "w") as f:
+        d = np.arange(12, dtype=dtype).reshape(shape) if data is None else data
+        f["d"] = d
+        f["g/x"] = np.arange(3)
+        if extra:
+            f["only_here"] = 1
+        if attr:
+            f["d"].attrs["u"] = 1
+
+
+base = np.arange(12, dtype="f8").reshape(3, 4)
+small("base.h5")
+small("same.h5")
+changed = base.copy()
+changed[0, 2] += 0.001
+changed[2, 3] += 0.001
+small("changed.h5", data=changed)
+small("extra.h5", extra=True)
+small("attr.h5", attr=True)
+small("reshaped.h5", shape=(4, 3))
+small("int.h5", dtype="i4")
+
+# One object under two names (a hard link) against two separate copies.
+with h5py.File(os.path.join(out, "hardlinked.h5"), "w") as f:
+    f["x"] = np.arange(5)
+    f["y"] = f["x"]
+    g = f.create_group("g")
+    g["d"] = np.arange(3)
+    g.create_group("s")["e"] = np.arange(2)
+    f["h"] = g
+for name, last in (("copied.h5", 1), ("copied_changed.h5", 9)):
+    with h5py.File(os.path.join(out, name), "w") as f:
+        f["x"] = np.arange(5)
+        f["y"] = np.arange(5)
+        for gname in ("g", "h"):
+            g = f.create_group(gname)
+            g["d"] = np.arange(3)
+            g.create_group("s")["e"] = np.array([0, last if gname == "h" else 1])
+
+# 64-bit integers one apart, beyond f64's 2^53 integer precision.
+for name, d in (("big1.h5", 0), ("big2.h5", 1)):
+    with h5py.File(os.path.join(out, name), "w") as f:
+        f["i"] = np.array([2**60 + d, -(2**62) - d], dtype="i8")
+        f["u"] = np.array([2**64 - 1 - d], dtype="u8")
+
+# Soft links: the same link targets, whose target objects differ.
+for name, v in (("soft1.h5", 0), ("soft2.h5", 1)):
+    with h5py.File(os.path.join(out, name), "w") as f:
+        f["z"] = np.arange(4) + v
+        f.create_group("g")["s"] = h5py.SoftLink("/z")
+        grp = f.create_group("grp")
+        grp["d"] = np.arange(3) + v
+        f["lnk"] = h5py.SoftLink("/grp")
+        f["dang"] = h5py.SoftLink("/nowhere")
+# Soft links: different link targets, whose target objects are equal.
+for name, t in (("target1.h5", "/a"), ("target2.h5", "/b")):
+    with h5py.File(os.path.join(out, name), "w") as f:
+        f["a"] = np.arange(4)
+        f["b"] = np.arange(4)
+        f["s"] = h5py.SoftLink(t)
+        f["rel"] = h5py.SoftLink("a")
+
+# Null-padded fixed strings with NULs, at the top level and nested in a
+# compound and in an array member.
+with h5py.File(os.path.join(out, "nulstrings.h5"), "w") as f:
+    f["top"] = np.array([b"", b"ab", b"a\x00b"], dtype="S3")
+    f["cmp"] = np.array(
+        [(b"", 1), (b"ab", 2), (b"a\x00b", 3)], dtype=[("s", "S3"), ("i", "i4")]
+    )
+    f["arr"] = np.array([([b"", b"x"],)], dtype=[("v", "(2,)S2")])
+
+with h5py.File(os.path.join(out, "userblock.h5"), "w", userblock_size=1024) as f:
+    f["d"] = np.arange(10)
+
+print(json.dumps(values))

crates/clawhdf5-tools/tests/h5rs_interop.rs

@@ -0,0 +1,775 @@
+//! `h5rs` against libhdf5's own tools, on files h5py writes
+//! (`tests/gen_files.py`): `ls` against h5ls, `stat` against h5stat, `dump`
+//! against h5dump, `diff` exit codes against h5diff, `dump --json` values
+//! against h5py, and `check` on valid and deliberately corrupted files.
+//!
+//! Needs python3 with h5py (`CLAWHDF5_PYTHON`) and, for the comparisons,
+//! the libhdf5 command-line tools (h5ls, h5stat, h5dump, h5diff) on `PATH`.
+//! Each test skips when what it needs is missing, unless
+//! `CLAWHDF5_REQUIRE_INTEROP=1`, which turns a skip into a failure.
+
+use std::collections::BTreeMap;
+use std::path::{Path, PathBuf};
+use std::process::{Command, Output};
+
+use clawhdf5_format::checksum::jenkins_lookup3;
+
+fn python() -> String {
+    std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
+}
+
+fn interop_required() -> bool {
+    std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
+}
+
+fn python_available() -> bool {
+    Command::new(python())
+        .args(["-c", "import h5py, numpy"])
+        .output()
+        .map(|o| o.status.success())
+        .unwrap_or(false)
+}
+
+fn tool_available(name: &str) -> bool {
+    Command::new(name)
+        .arg("--version")
+        .output()
+        .map(|o| o.status.success())
+        .unwrap_or(false)
+}
+
+/// Skip (return true) when `what` is unavailable, or fail when interop is
+/// required.
+fn missing(ok: bool, what: &str) -> bool {
+    if ok {
+        return false;
+    }
+    assert!(
+        !interop_required(),
+        "CLAWHDF5_REQUIRE_INTEROP=1 but {what} is not available"
+    );
+    eprintln!("SKIP: {what} not available");
+    true
+}
+
+fn h5rs(args: &[&str]) -> Output {
+    Command::new(env!("CARGO_BIN_EXE_h5rs"))
+        .args(args)
+        .output()
+        .expect("run h5rs")
+}
+
+fn run(tool: &str, args: &[&str]) -> Output {
+    Command::new(tool).args(args).output().expect("run tool")
+}
+
+fn stdout(o: &Output) -> String {
+    String::from_utf8_lossy(&o.stdout).into_owned()
+}
+
+/// The generated files and what h5py read from them.
+struct Files {
+    dir: tempfile::TempDir,
+    values: serde_json::Value,
+}
+
+impl Files {
+    fn path(&self, name: &str) -> PathBuf {
+        self.dir.path().join(name)
+    }
+
+    fn p(&self, name: &str) -> String {
+        self.path(name).to_string_lossy().into_owned()
+    }
+}
+
+fn generate() -> Option<Files> {
+    if missing(python_available(), "python3 with h5py") {
+        return None;
+    }
+    let dir = tempfile::tempdir().unwrap();
+    let script = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/gen_files.py");
+    let out = Command::new(python())
+        .arg(&script)
+        .arg(dir.path())
+        .output()
+        .expect("run gen_files.py");
+    assert!(
+        out.status.success(),
+        "gen_files.py failed:\n{}",
+        String::from_utf8_lossy(&out.stderr)
+    );
+    let values = serde_json::from_slice(&out.stdout).expect("gen_files.py output");
+    Some(Files { dir, values })
+}
+
+const LIBVERS: [&str; 2] = ["earliest.h5", "latest.h5"];
+
+// ---------------------------------------------------------------------------
+// ls
+// ---------------------------------------------------------------------------
+
+/// Every h5ls line must be the start of the matching h5rs line (h5rs adds
+/// the datatype after a dataset's shape).
+fn assert_ls_like(ours: &str, reference: &str, what: &str) {
+    let ours: Vec<&str> = ours.lines().collect();
+    let refs: Vec<&str> = reference.lines().collect();
+    assert_eq!(
+        ours.len(),
+        refs.len(),
+        "{what}: line count\nh5rs:\n{}\nh5ls:\n{}",
+        ours.join("\n"),
+        refs.join("\n")
+    );
+    for (o, r) in ours.iter().zip(&refs) {
+        let r = r.trim_end();
+        assert!(
+            o.starts_with(r) && (o.len() == r.len() || r.contains("Dataset {")),
+            "{what}:\n  h5rs: {o}\n  h5ls: {r}"
+        );
+    }
+}
+
+#[test]
+fn ls_matches_h5ls() {
+    let Some(f) = generate() else { return };
+    if missing(tool_available("h5ls"), "h5ls") {
+        return;
+    }
+    for name in LIBVERS.iter().chain(&["userblock.h5"]) {
+        let p = f.p(name);
+        for (args, what) in [(vec!["-r"], "recursive"), (vec![], "root")] {
+            let mut a: Vec<&str> = args.clone();
+            a.push(&p);
+            let ours = h5rs(&[&["ls"], a.as_slice()].concat());
+            assert!(ours.status.success(), "{name} {what}: {:?}", ours);
+            let reference = run("h5ls", &a);
+            assert_ls_like(
+                &stdout(&ours),
+                &stdout(&reference),
+                &format!("{name} {what}"),
+            );
+        }
+    }
+    // A group, and a dataset, named after the file.
+    for obj in ["/grp", "/grp/sub", "/contig", "/grp/ext2"] {
+        let arg = format!("{}{obj}", f.p("latest.h5"));
+        let ours = h5rs(&["ls", &arg]);
+        let reference = run("h5ls", &[&arg]);
+        assert_ls_like(&stdout(&ours), &stdout(&reference), &arg);
+        let ours = h5rs(&["ls", "-r", &arg]);
+        let reference = run("h5ls", &["-r", &arg]);
+        assert_ls_like(&stdout(&ours), &stdout(&reference), &format!("-r {arg}"));
+    }
+}
+
+#[test]
+fn ls_verbose_describes_layout_filters_and_attributes() {
+    let Some(f) = generate() else { return };
+    let o = h5rs(&["ls", "-r", "-v", &f.p("latest.h5")]);
+    assert!(o.status.success());
+    let s = stdout(&o);
+    for want in [
+        "Layout:    chunked (extensible array index)",
+        "Layout:    chunked (v2 B-tree index)",
+        "Layout:    chunked (fixed array index)",
+        "Layout:    chunked (single chunk index)",
+        "Filter-0:  shuffle-2",
+        "Filter-1:  deflate-1",
+        "Filter-2:  fletcher32-3",
+        "Chunks:    {100} 400 bytes",
+        "Attribute: units scalar",
+        "Data:      \"m\"",
+    ] {
+        assert!(s.contains(want), "missing {want:?} in\n{s}");
+    }
+    let o = h5rs(&["ls", "-v", &format!("{}/grp", f.p("earliest.h5"))]);
+    assert!(stdout(&o).contains("Layout:    chunked (v1 B-tree index)"));
+}
+
+// ---------------------------------------------------------------------------
+// stat
+// ---------------------------------------------------------------------------
+
+/// `key: value` lines of an h5stat-style report.
+fn stat_facts(s: &str) -> BTreeMap<String, String> {
+    s.lines()
+        .filter_map(|l| {
+            let (k, v) = l.rsplit_once(':')?;
+            Some((k.trim().to_string(), v.trim().to_string()))
+        })
+        .collect()
+}
+
+#[test]
+fn stat_matches_h5stat() {
+    let Some(f) = generate() else { return };
+    if missing(tool_available("h5stat"), "h5stat") {
+        return;
+    }
+    let keys = [
+        "# of unique groups",
+        "# of unique datasets",
+        "# of unique named datatypes",
+        "# of unique links",
+        "# of unique other",
+        "Max. # of links to object",
+        "Max. # of objects in group",
+        "Max. rank of datasets",
+        "Max. dimension size of 1-D datasets",
+        "Total raw data size",
+        "Dataset layout counts[COMPACT]",
+        "Dataset layout counts[CONTIG]",
+        "Dataset layout counts[CHUNKED]",
+        "Dataset layout counts[VIRTUAL]",
+        "NO filter",
+        "GZIP filter",
+        "SHUFFLE filter",
+        "FLETCHER32 filter",
+        "SZIP filter",
+        "NBIT filter",
+        "SCALEOFFSET filter",
+        "USER-DEFINED filter",
+        "Max. # of attributes to objects",
+        "Total space",
+    ];
+    for name in LIBVERS.iter().chain(&["userblock.h5", "base.h5"]) {
+        let p = f.p(name);
+        let ours = h5rs(&["stat", &p]);
+        assert!(ours.status.success(), "{name}: {ours:?}");
+        let reference = run("h5stat", &[&p]);
+        assert!(reference.status.success(), "h5stat {name} failed");
+        let (o, r) = (stat_facts(&stdout(&ours)), stat_facts(&stdout(&reference)));
+        for k in keys {
+            // h5stat leaves out the attribute summary when nothing has one.
+            let Some(want) = r.get(k) else { continue };
+            assert_eq!(o.get(k), Some(want), "{name}: {k}");
+        }
+    }
+}
+
+// ---------------------------------------------------------------------------
+// dump
+// ---------------------------------------------------------------------------
+
+#[test]
+fn dump_matches_h5dump() {
+    let Some(f) = generate() else { return };
+    if missing(tool_available("h5dump"), "h5dump") {
+        return;
+    }
+    for name in LIBVERS.iter().chain(&["userblock.h5", "changed.h5"]) {
+        let p = f.p(name);
+        for args in [vec![], vec!["-A"]] {
+            let mut a = args.clone();
+            a.push(p.as_str());
+            let ours = h5rs(&[&["dump"], a.as_slice()].concat());
+            assert!(ours.status.success(), "{name} {args:?}: {ours:?}");
+            let reference = run("h5dump", &a);
+            // h5dump names the file as given; h5rs by its file name.
+            let r = stdout(&reference).replacen(&p, name, 1);
+            assert_eq!(stdout(&ours), r, "{name} {args:?}");
+        }
+    }
+    let p = f.p("latest.h5");
+    let ours = h5rs(&["dump", "-d", "/grp/ext2", &p]);
+    let reference = run("h5dump", &["-d", "/grp/ext2", &p]);
+    assert_eq!(
+        stdout(&ours),
+        stdout(&reference).replacen(&p, "latest.h5", 1)
+    );
+}
+
+/// A null-padded fixed string prints every byte (NULs as `\000`), as
+/// h5dump prints it, also inside a compound and an array member.
+#[test]
+fn dump_shows_nul_padding_in_nested_strings() {
+    let Some(f) = generate() else { return };
+    let p = f.p("nulstrings.h5");
+    let ours = stdout(&h5rs(&["dump", &p]));
+    for want in [
+        r#"(0): "\000\000\000", "ab\000", "a\000b""#,
+        r#""\000\000\000","#,
+        r#""a\000b","#,
+        r#"[ "\000\000", "x\000" ]"#,
+    ] {
+        assert!(ours.contains(want), "no {want} in\n{ours}");
+    }
+    if missing(tool_available("h5dump"), "h5dump") {
+        return;
+    }
+    let reference = run("h5dump", &[&p]);
+    assert_eq!(ours, stdout(&reference).replacen(&p, "nulstrings.h5", 1));
+}
+
+#[test]
+fn dump_json_values_match_h5py() {
+    let Some(f) = generate() else { return };
+    for name in LIBVERS {
+        let o = h5rs(&["dump", "--json", &f.p(name)]);
+        assert!(o.status.success(), "{name}: {o:?}");
+        let doc: serde_json::Value = serde_json::from_slice(&o.stdout).unwrap();
+        assert_eq!(doc["apiVersion"], "1.1.1");
+        let root = doc["root"].as_str().unwrap();
+        assert!(doc["groups"][root].is_object());
+        let mut by_path = BTreeMap::new();
+        for (_, d) in doc["datasets"].as_object().unwrap() {
+            for a in d["alias"].as_array().unwrap() {
+                by_path.insert(a.as_str().unwrap().to_string(), d.clone());
+            }
+        }
+        let want = f.values[name.trim_end_matches(".h5")].as_object().unwrap();
+        assert!(want.len() > 10);
+        for (path, v) in want {
+            let d = by_path
+                .get(path)
+                .unwrap_or_else(|| panic!("{name}: no {path}"));
+            let got = &d["value"];
+            // JSON numbers: compare as f64 (h5py's ints and floats both
+            // round-trip exactly through JSON here).
+            assert_eq!(flatten(got), flatten(v), "{name}: {path}");
+        }
+        let links = doc["groups"][root]["links"].as_array().unwrap();
+        let soft = links.iter().find(|l| l["title"] == "soft").unwrap();
+        assert_eq!(soft["class"], "H5L_TYPE_SOFT");
+        assert_eq!(soft["h5path"], "/contig");
+        let ext = links.iter().find(|l| l["title"] == "external").unwrap();
+        assert_eq!(ext["class"], "H5L_TYPE_EXTERNAL");
+        assert_eq!(ext["file"], "other.h5");
+    }
+}
+
+fn flatten(v: &serde_json::Value) -> Vec<f64> {
+    match v {
+        serde_json::Value::Array(a) => a.iter().flat_map(flatten).collect(),
+        serde_json::Value::Number(n) => vec![n.as_f64().unwrap()],
+        other => panic!("not numeric: {other}"),
+    }
+}
+
+// ---------------------------------------------------------------------------
+// diff
+// ---------------------------------------------------------------------------
+
+fn code(o: &Output) -> i32 {
+    o.status.code().expect("exit code")
+}
+
+#[test]
+fn diff_exit_codes_match_h5diff() {
+    let Some(f) = generate() else { return };
+    if missing(tool_available("h5diff"), "h5diff") {
+        return;
+    }
+    let (b, same, changed, extra, attr) = (
+        f.p("base.h5"),
+        f.p("same.h5"),
+        f.p("changed.h5"),
+        f.p("extra.h5"),
+        f.p("attr.h5"),
+    );
+    let missing_file = f.p("does-not-exist.h5");
+    let cases: Vec<(Vec<&str>, i32)> = vec![
+        (vec![&b, &same], 0),
+        (vec![&b, &b], 0),
+        (vec![&b, &changed], 1),
+        (vec![&b, &changed, "/d"], 1),
+        (vec![&b, &changed, "/g"], 0),
+        (vec!["-d", "0.01", &b, &changed], 0),
+        (vec!["-d", "0.0001", &b, &changed], 1),
+        (vec!["-p", "0.01", &b, &changed], 0),
+        (vec!["-p", "1e-9", &b, &changed], 1),
+        (vec![&b, &extra], 1),
+        (vec![&b, &attr], 1),
+        (vec![&b, &attr, "/g"], 0),
+        (vec![&b, &same, "/nope"], 2),
+        (vec![&b, &missing_file], 2),
+        (vec![&b, &same, "/d", "/d"], 0),
+    ];
+    for (args, want) in cases {
+        let theirs = code(&run("h5diff", &args));
+        assert_eq!(theirs, want, "h5diff {args:?} (test expectation)");
+        let o = h5rs(&[&["diff"], args.as_slice()].concat());
+        assert_eq!(code(&o), want, "h5rs diff {args:?}: {}", stdout(&o));
+        let q = h5rs(&[&["diff", "-q"], args.as_slice()].concat());
+        assert_eq!(code(&q), want, "h5rs diff -q {args:?}");
+        assert!(q.stdout.is_empty(), "-q printed output");
+    }
+    // The report lists the differing positions.
+    let o = h5rs(&["diff", "-r", &b, &changed, "/d"]);
+    let s = stdout(&o);
+    assert!(s.contains("[ 0 2 ]") && s.contains("[ 2 3 ]"), "{s}");
+    assert!(s.contains("2 difference(s) found"), "{s}");
+}
+
+/// The option names are h5diff's: -c is --compare (a flag), the count is
+/// -n/--count, and the --name=value forms are accepted.
+#[test]
+fn diff_options_are_named_like_h5diff() {
+    let Some(f) = generate() else { return };
+    let (b, changed) = (f.p("base.h5"), f.p("changed.h5"));
+    let cases: Vec<(Vec<&str>, i32)> = vec![
+        // "2" is then the first file name, which does not exist.
+        (vec!["-r", "-c", "2", &b, &changed], 2),
+        (vec!["-c", &b, &changed], 1),
+        (vec!["--compare", &b, &b], 0),
+        (vec!["-r", "-n", "1", &b, &changed], 1),
+        (vec!["-r", "--count=1", &b, &changed], 1),
+        (vec!["--delta=0.01", &b, &changed], 0),
+        (vec!["--relative=1e-9", &b, &changed], 1),
+    ];
+    let have_h5diff = tool_available("h5diff");
+    for (args, want) in cases {
+        if have_h5diff {
+            assert_eq!(code(&run("h5diff", &args)), want, "h5diff {args:?}");
+        }
+        let o = h5rs(&[&["diff"], args.as_slice()].concat());
+        assert_eq!(code(&o), want, "h5rs diff {args:?}: {}", stdout(&o));
+    }
+    for count in [vec!["-n", "1"], vec!["--count=1"]] {
+        let o = h5rs(&[&["diff", "-r"], count.as_slice(), &[&b, &changed, "/d"]].concat());
+        let s = stdout(&o);
+        assert!(
+            s.contains("[ 0 2 ]") && !s.contains("[ 2 3 ]"),
+            "{count:?}: {s}"
+        );
+        assert!(s.contains("2 difference(s) found"), "{s}");
+    }
+    assert_eq!(code(&h5rs(&["diff", "--bogus=1", &b, &b])), 2);
+}
+
+/// Tolerances on 64-bit integers one apart, far beyond f64's integer
+/// precision: compared exactly, as h5diff does.
+#[test]
+fn diff_tolerances_compare_large_integers_exactly() {
+    let Some(f) = generate() else { return };
+    let (a, b) = (f.p("big1.h5"), f.p("big2.h5"));
+    let cases: Vec<(Vec<&str>, i32)> = vec![
+        (vec![&a, &b], 1),
+        (vec!["-d", "0", &a, &b], 1),
+        (vec!["-d", "0.5", &a, &b], 1),
+        (vec!["-d", "1", &a, &b], 0),
+        (vec!["-d", "0", &a, &b, "/u"], 1),
+        (vec!["-p", "0", &a, &b], 1),
+        (vec!["-p", "1e-18", &a, &b, "/i"], 1),
+        (vec!["-p", "1e-15", &a, &b], 0),
+    ];
+    let have_h5diff = tool_available("h5diff");
+    for (args, want) in cases {
+        if have_h5diff {
+            assert_eq!(code(&run("h5diff", &args)), want, "h5diff {args:?}");
+        }
+        let o = h5rs(&[&["diff"], args.as_slice()].concat());
+        assert_eq!(code(&o), want, "h5rs diff {args:?}: {}", stdout(&o));
+    }
+    // The reported difference is exact too.
+    let s = stdout(&h5rs(&["diff", "-r", "-d", "0", &a, &b, "/u"]));
+    assert!(
+        s.contains("18446744073709551615    18446744073709551614    1\n"),
+        "{s}"
+    );
+}
+
+/// A soft link is compared as a link (its target path), as h5diff does,
+/// unless `--follow-symlinks`, which compares (and walks into) what it
+/// leads to; two dangling links are then the same.
+#[test]
+fn diff_soft_links_like_h5diff() {
+    let Some(f) = generate() else { return };
+    let (s1, s2, t1, t2) = (
+        f.p("soft1.h5"),
+        f.p("soft2.h5"),
+        f.p("target1.h5"),
+        f.p("target2.h5"),
+    );
+    let fl = "--follow-symlinks";
+    let cases: Vec<(Vec<&str>, i32)> = vec![
+        (vec![&s1, &s2, "/g/s"], 0),
+        (vec![&s1, &s2, "/g"], 0),
+        (vec![&s1, &s2, "/lnk"], 0),
+        (vec![&s1, &s2, "/dang"], 0),
+        (vec![&s1, &s2, "/lnk/d"], 1),
+        (vec![&t1, &t2, "/s"], 1),
+        (vec![&t1, &t2, "/s", "/rel"], 1),
+        (vec![fl, &s1, &s2, "/g/s"], 1),
+        (vec![fl, &s1, &s2, "/g"], 1),
+        (vec![fl, &s1, &s2, "/lnk"], 1),
+        (vec![fl, &s1, &s2, "/dang"], 0),
+        (vec![fl, &s1, &s1], 0),
+        (vec![fl, &t1, &t2, "/s"], 0),
+        (vec![fl, &t1, &t2, "/s", "/rel"], 0),
+    ];
+    let have_h5diff = tool_available("h5diff");
+    for (args, want) in cases {
+        if have_h5diff {
+            assert_eq!(code(&run("h5diff", &args)), want, "h5diff {args:?}");
+        }
+        let o = h5rs(&[&["diff"], args.as_slice()].concat());
+        assert_eq!(code(&o), want, "h5rs diff {args:?}: {}", stdout(&o));
+    }
+}
+
+/// An object hard-linked under two names is compared under both, with
+/// everything below it, so a file that shares one object between two names
+/// equals a file that stores two identical copies.
+#[test]
+fn diff_compares_every_hard_link_path() {
+    let Some(f) = generate() else { return };
+    let (linked, copied, changed) = (
+        f.p("hardlinked.h5"),
+        f.p("copied.h5"),
+        f.p("copied_changed.h5"),
+    );
+    // A hard-linked dataset: h5diff agrees (exit 0).
+    for (a, b) in [(&linked, &copied), (&copied, &linked)] {
+        let o = h5rs(&["diff", a, b, "/y"]);
+        assert_eq!(code(&o), 0, "{a} {b} /y: {}", stdout(&o));
+        if tool_available("h5diff") {
+            assert_eq!(code(&run("h5diff", &[a, b, "/y"])), 0, "h5diff /y");
+        }
+        // The whole file, including a hard-linked group's members (h5diff
+        // does not descend a group's second name, so it reports those
+        // members as present in one file only and exits 1).
+        let o = h5rs(&["diff", a, b]);
+        assert_eq!(code(&o), 0, "{a} {b}: {}", stdout(&o));
+        assert!(!stdout(&o).contains("exists only"), "{}", stdout(&o));
+    }
+    // A value under the second name is compared, not skipped.
+    let o = h5rs(&["diff", "-r", &linked, &changed]);
+    assert_eq!(code(&o), 1, "{}", stdout(&o));
+    assert!(
+        stdout(&o).contains("dataset: </h/s/e> and </h/s/e>"),
+        "{}",
+        stdout(&o)
+    );
+    assert!(!stdout(&o).contains("</g/s/e>"), "{}", stdout(&o));
+}
+
+/// Where h5rs deliberately differs from h5diff: objects that cannot be
+/// compared (another shape, another datatype class) are a difference.
+#[test]
+fn diff_counts_incomparable_objects_as_different() {
+    let Some(f) = generate() else { return };
+    let b = f.p("base.h5");
+    for other in ["reshaped.h5", "int.h5"] {
+        let o = h5rs(&["diff", &b, &f.p(other)]);
+        assert_eq!(code(&o), 1, "{other}: {}", stdout(&o));
+        assert!(stdout(&o).contains("Not comparable"), "{other}");
+    }
+}
+
+// ---------------------------------------------------------------------------
+// check
+// ---------------------------------------------------------------------------
+
+#[test]
+fn check_accepts_valid_files() {
+    let Some(f) = generate() else { return };
+    for name in LIBVERS.iter().chain(&[
+        "userblock.h5",
+        "base.h5",
+        "changed.h5",
+        "extra.h5",
+        "attr.h5",
+    ]) {
+        let o = h5rs(&["check", "--data", &f.p(name)]);
+        assert_eq!(code(&o), 0, "{name}:\n{}", stdout(&o));
+        assert!(stdout(&o).contains("no problems found"));
+    }
+    // The newest format's checksummed structures were all verified.
+    let s = stdout(&h5rs(&["check", &f.p("latest.h5")]));
+    assert!(s.contains("superblock 1, v2 object headers 33,"), "{s}");
+    assert!(s.contains("chunk indexes 5"), "{s}");
+}
+
+/// Signatures of checksummed structures, and where each one's checksum
+/// is: found by trying every end offset (the checksum is the Jenkins
+/// lookup3 hash of the bytes before it). A fractal heap direct block's
+/// checksum is instead stored near its start, computed over the whole block
+/// with the field zeroed.
+fn checksummed_structures(data: &[u8]) -> Vec<(&'static str, usize, usize)> {
+    const SIGS: [&str; 13] = [
+        "OHDR", "OCHK", "BTHD", "BTIN", "BTLF", "FRHP", "FHIB", "EAHD", "EAIB", "EASB", "EADB",
+        "FAHD", "FADB",
+    ];
+    let mut found = Vec::new();
+    for start in 0..data.len().saturating_sub(4) {
+        let sig = &data[start..start + 4];
+        if let Some(name) = SIGS.iter().find(|s| s.as_bytes() == sig) {
+            let limit = data.len().min(start + 65536);
+            for end in start + 8..limit.saturating_sub(4) {
+                let stored = u32::from_le_bytes(data[end..end + 4].try_into().unwrap());
+                if jenkins_lookup3(&data[start..end]) == stored {
+                    found.push((*name, start, end));
+                    break;
+                }
+            }
+        } else if sig == b"FHDB" {
+            // version(1) heap address(8) block offset(1..8) checksum(4)
+            'found: for boff in 1..=8usize {
+                let at = start + 5 + 8 + boff;
+                let Some(stored) = data.get(at..at + 4) else {
+                    break;
+                };
+                let stored = u32::from_le_bytes(stored.try_into().unwrap());
+                for size in [512usize, 1024, 2048, 4096, 8192, 16384, 32768, 65536] {
+                    let Some(block) = data.get(start..start + size) else {
+                        break;
+                    };
+                    let mut b = block.to_vec();
+                    b[at - start..at - start + 4].fill(0);
+                    if jenkins_lookup3(&b) == stored {
+                        found.push(("FHDB", start, at));
+                        break 'found;
+                    }
+                }
+            }
+        }
+    }
+    found
+}
+
+#[test]
+fn check_flags_every_corrupted_checksum() {
+    let Some(f) = generate() else { return };
+    let path = f.path("latest.h5");
+    let data = std::fs::read(&path).unwrap();
+    let structures = checksummed_structures(&data);
+    let kinds: std::collections::BTreeSet<&str> = structures.iter().map(|s| s.0).collect();
+    for want in [
+        "OHDR", "OCHK", "BTHD", "BTLF", "FRHP", "FHDB", "EAHD", "EAIB", "EADB", "FAHD", "FADB",
+    ] {
+        assert!(kinds.contains(want), "test file has no {want}: {kinds:?}");
+    }
+    let bad = f.path("bad.h5");
+    for (name, start, at) in structures {
+        let mut d = data.clone();
+        d[at] ^= 0x01;
+        std::fs::write(&bad, &d).unwrap();
+        let o = h5rs(&["check", &bad.to_string_lossy()]);
+        let s = stdout(&o);
+        assert_eq!(
+            code(&o),
+            1,
+            "{name} at {start:#x}: checksum flip not flagged\n{s}"
+        );
+        assert!(
+            s.lines()
+                .any(|l| l.starts_with("problem: 0x") && l.contains("checksum")),
+            "{name} at {start:#x}: no checksum problem reported\n{s}"
+        );
+    }
+}
+
+/// `check --data` follows variable-length elements into the global heap:
+/// a damaged collection is reported at its address, for the dataset and
+/// the attribute that point into it. Without --data it is not read.
+#[test]
+fn check_data_follows_vl_data_into_the_global_heap() {
+    let Some(f) = generate() else { return };
+    for name in LIBVERS {
+        let mut data = std::fs::read(f.path(name)).unwrap();
+        let gcols: Vec<usize> = (0..data.len() - 4)
+            .filter(|&i| &data[i..i + 4] == b"GCOL")
+            .collect();
+        assert!(!gcols.is_empty(), "{name}: no global heap");
+        // "GCOL" version(1) reserved(3) size(8), then heap object 1:
+        // index(2) refcount(2) reserved(4) size(8) — claim 4 GiB.
+        let at = gcols[0];
+        data[at + 24..at + 28].fill(0xff);
+        let bad = f.path(&format!("bad-gcol-{name}"));
+        std::fs::write(&bad, &data).unwrap();
+        let bad = bad.to_string_lossy().into_owned();
+        let o = h5rs(&["check", "--data", &bad]);
+        let s = stdout(&o);
+        assert_eq!(code(&o), 1, "{name}: {s}");
+        let want = format!("problem: {at:#x} /vlstr: variable-length data: global heap");
+        assert!(s.contains(&want), "{name}: no {want:?} in\n{s}");
+        // h5dump refuses the file too.
+        if tool_available("h5dump") {
+            assert_ne!(code(&run("h5dump", &[&bad])), 0, "{name}: h5dump read it");
+        }
+        assert_eq!(code(&h5rs(&["check", &bad])), 0, "{name}: without --data");
+        let ok = h5rs(&["check", "--data", &f.p(name)]);
+        assert!(
+            stdout(&ok).contains(&format!("global heap collections read: {}", gcols.len())),
+            "{name}: {}",
+            stdout(&ok)
+        );
+    }
+}
+
+#[test]
+fn check_flags_a_truncated_file() {
+    let Some(f) = generate() else { return };
+    let data = std::fs::read(f.path("latest.h5")).unwrap();
+    let bad = f.path("truncated.h5");
+    std::fs::write(&bad, &data[..data.len() * 3 / 4]).unwrap();
+    let o = h5rs(&["check", &bad.to_string_lossy()]);
+    assert_eq!(code(&o), 1);
+    assert!(stdout(&o).contains("file is truncated"), "{}", stdout(&o));
+}
+
+/// A v1 B-tree chunk index has no checksum; its keys are checked against
+/// the dataset: a chunk offset that is not a multiple of the chunk size.
+#[test]
+fn check_flags_a_misaligned_chunk() {
+    let Some(f) = generate() else { return };
+    let path = f.path("earliest.h5");
+    let mut data = std::fs::read(&path).unwrap();
+    // The leaf chunk B-tree of /grp/gz (10 deflated chunks of 100 i32, so
+    // each stored in under 400 bytes): "TREE" type(1)=1 level(1)=0
+    // entries(2) left(8) right(8), then key, child, key, child, ... where a
+    // rank-1 chunk key is size(4) mask(4) offsets(2 x 8) and a child is 8.
+    // Key 1 starts at 24 + 24 + 8 = 56; its dimension-0 offset (100) at 64.
+    let mut patched = false;
+    for start in 0..data.len() - 72 {
+        if &data[start..start + 4] == b"TREE" && data[start + 4] == 1 && data[start + 5] == 0 {
+            let entries = u16::from_le_bytes(data[start + 6..start + 8].try_into().unwrap());
+            let size0 = u32::from_le_bytes(data[start + 24..start + 28].try_into().unwrap());
+            let off1 = u64::from_le_bytes(data[start + 64..start + 72].try_into().unwrap());
+            if entries == 10 && size0 < 400 && off1 == 100 {
+                data[start + 64..start + 72].copy_from_slice(&103u64.to_le_bytes());
+                patched = true;
+                break;
+            }
+        }
+    }
+    assert!(patched, "no chunk B-tree found for /grp/gz");
+    let bad = f.path("misaligned.h5");
+    std::fs::write(&bad, &data).unwrap();
+    let o = h5rs(&["check", &bad.to_string_lossy()]);
+    let s = stdout(&o);
+    assert_eq!(code(&o), 1, "{s}");
+    // The library's chunk-index reader refuses the key first (as libhdf5
+    // does); either way the problem is reported against the dataset.
+    assert!(
+        s.contains("/grp/gz: chunk at [103, 0] offset 103 in dimension 0 is not a multiple of the chunk size 100")
+            || s.contains("/grp/gz: chunk index (v1 B-tree): chunked read error: bad coordinate offset [103, 0] for chunk dimensions [100, 4]"),
+        "{s}"
+    );
+}
+
+#[test]
+fn every_subcommand_rejects_a_non_hdf5_file_cleanly() {
+    let dir = tempfile::tempdir().unwrap();
+    let p = dir.path().join("junk.h5");
+    std::fs::write(&p, b"this is not an HDF5 file at all").unwrap();
+    let p = p.to_string_lossy().into_owned();
+    for args in [
+        vec!["ls", p.as_str()],
+        vec!["dump", p.as_str()],
+        vec!["stat", p.as_str()],
+        vec!["diff", p.as_str(), p.as_str()],
+    ] {
+        let o = h5rs(&args);
+        assert_eq!(code(&o), 2, "{args:?}");
+        assert!(String::from_utf8_lossy(&o.stderr).contains("not an HDF5 file"));
+    }
+    let o = h5rs(&["check", &p]);
+    assert_eq!(code(&o), 1);
+    assert!(stdout(&o).contains("no HDF5 signature"));
+    assert_eq!(code(&h5rs(&["nonsense"])), 2);
+    assert_eq!(code(&h5rs(&["ls"])), 2);
+    assert_eq!(code(&h5rs(&["--help"])), 0);
+}

crates/clawhdf5-wasm/Cargo.toml

@@ -0,0 +1,30 @@
+[package]
+name = "clawhdf5-wasm"
+version.workspace = true
+edition.workspace = true
+rust-version.workspace = true
+description = "Read HDF5 and NetCDF-4 files in the browser: clawhdf5's reader compiled to WebAssembly"
+license.workspace = true
+repository.workspace = true
+keywords = ["hdf5", "netcdf", "wasm", "browser"]
+categories = ["wasm", "parser-implementations", "science"]
+# Distributed as the wasm-bindgen package built by examples/wasm-viewer/build.sh.
+publish = false
+
+[lib]
+# cdylib for wasm-bindgen; rlib so the pure-Rust core is tested natively.
+crate-type = ["cdylib", "rlib"]
+
+[dependencies]
+# No mmap (there is no file system) and no `parallel` (no threads on
+# wasm32-unknown-unknown). lz4 is pure Rust; zstd and szip link C and are
+# left out, so such datasets fail with a clear filter error.
+clawhdf5 = { path = "../clawhdf5", version = "2.7.0", default-features = false, features = ["lz4"] }
+clawhdf5-format = { path = "../clawhdf5-format", version = "2.7.0" }
+# Must match the wasm-bindgen CLI exactly; build.sh checks.
+wasm-bindgen = "0.2.129"
+js-sys = "0.3.106"
+
+[dev-dependencies]
+serde_json = "1"
+tempfile = { workspace = true }

crates/clawhdf5-wasm/src/core.rs

@@ -0,0 +1,603 @@
+//! The reader behind the JavaScript API, in plain Rust so it is tested
+//! natively. The `wasm_bindgen` layer in `lib.rs` only converts these types
+//! to JavaScript values.
+//!
+//! Every read either returns the dataset's values or an error: a datatype
+//! with no typed-array mapping (compound, reference, opaque, ...) is refused
+//! with a message naming it, never returned as reinterpreted bytes.
+
+use clawhdf5::{AttrValue, File, Selection};
+use clawhdf5_format::data_read;
+use clawhdf5_format::datatype::{Datatype, DatatypeByteOrder};
+
+/// Errors are reported to JavaScript as messages.
+pub type Result<T> = std::result::Result<T, String>;
+
+fn err(e: impl std::fmt::Display) -> String {
+    e.to_string()
+}
+
+/// What a path names.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Kind {
+    Group,
+    Dataset,
+}
+
+impl Kind {
+    pub fn as_str(self) -> &'static str {
+        match self {
+            Kind::Group => "group",
+            Kind::Dataset => "dataset",
+        }
+    }
+}
+
+/// One entry of a group listing.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct Child {
+    pub name: String,
+    pub kind: Kind,
+}
+
+/// A dataset's metadata.
+#[derive(Debug, Clone, PartialEq)]
+pub struct DatasetInfo {
+    /// Dataspace dimensions (empty for a scalar).
+    pub shape: Vec<u64>,
+    /// Maximum dimensions, `None` per unlimited dimension; `None` overall
+    /// when the dataspace records none.
+    pub maxshape: Option<Vec<Option<u64>>>,
+    /// Human-readable datatype, e.g. `f64`, `i16 (big-endian)`, `string[8]`.
+    pub dtype: String,
+    /// Dimensions of an array datatype's elements, appended to the shape of
+    /// what [`Reader::read`] returns (empty otherwise).
+    pub element_shape: Vec<u64>,
+}
+
+/// Decoded values, one variant per JavaScript typed array.
+#[derive(Debug, Clone, PartialEq)]
+pub enum Data {
+    F32(Vec<f32>),
+    F64(Vec<f64>),
+    I8(Vec<i8>),
+    I16(Vec<i16>),
+    I32(Vec<i32>),
+    I64(Vec<i64>),
+    U8(Vec<u8>),
+    U16(Vec<u16>),
+    U32(Vec<u32>),
+    U64(Vec<u64>),
+    /// Fixed- and variable-length strings, and enumeration member names.
+    Strings(Vec<String>),
+}
+
+impl Data {
+    pub fn len(&self) -> usize {
+        match self {
+            Data::F32(v) => v.len(),
+            Data::F64(v) => v.len(),
+            Data::I8(v) => v.len(),
+            Data::I16(v) => v.len(),
+            Data::I32(v) => v.len(),
+            Data::I64(v) => v.len(),
+            Data::U8(v) => v.len(),
+            Data::U16(v) => v.len(),
+            Data::U32(v) => v.len(),
+            Data::U64(v) => v.len(),
+            Data::Strings(v) => v.len(),
+        }
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.len() == 0
+    }
+}
+
+/// Values in row-major order with their shape.
+#[derive(Debug, Clone, PartialEq)]
+pub struct Array {
+    pub shape: Vec<u64>,
+    pub data: Data,
+}
+
+/// A regular hyperslab, as in `H5Sselect_hyperslab`. `stride` and `block`
+/// default to 1 in every dimension.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct Hyperslab {
+    pub start: Vec<u64>,
+    pub count: Vec<u64>,
+    pub stride: Option<Vec<u64>>,
+    pub block: Option<Vec<u64>>,
+}
+
+/// An attribute: its value, or why it has none.
+#[derive(Debug, Clone)]
+pub struct Attr {
+    pub name: String,
+    pub value: AttrValue,
+}
+
+/// An open file, held in memory.
+pub struct Reader {
+    file: File,
+}
+
+impl Reader {
+    /// Parse a file from its bytes (the browser hands over the whole file).
+    pub fn open(bytes: Vec<u8>) -> Result<Self> {
+        Ok(Self {
+            file: File::from_bytes(bytes).map_err(err)?,
+        })
+    }
+
+    /// Whether `path` names a group or a dataset.
+    pub fn kind(&self, path: &str) -> Result<Kind> {
+        match self.file.dataset(path) {
+            Ok(_) => Ok(Kind::Dataset),
+            Err(clawhdf5::Error::NotADataset(_)) => Ok(Kind::Group),
+            Err(e) => Err(err(e)),
+        }
+    }
+
+    /// The groups, then the datasets, in the group at `path` (`/` is the
+    /// root). Soft links are listed as their targets; external and dangling
+    /// links, and named datatypes, are left out.
+    pub fn list(&self, path: &str) -> Result<Vec<Child>> {
+        if self.kind(path)? != Kind::Group {
+            return Err(format!("not a group: {path}"));
+        }
+        let group = self.file.group(path).map_err(err)?;
+        let mut out: Vec<Child> = group
+            .groups()
+            .map_err(err)?
+            .into_iter()
+            .map(|name| Child {
+                name,
+                kind: Kind::Group,
+            })
+            .collect();
+        out.extend(
+            group
+                .datasets()
+                .map_err(err)?
+                .into_iter()
+                .map(|name| Child {
+                    name,
+                    kind: Kind::Dataset,
+                }),
+        );
+        Ok(out)
+    }
+
+    /// Shape, max shape and datatype of the dataset at `path`.
+    pub fn info(&self, path: &str) -> Result<DatasetInfo> {
+        let ds = self.file.dataset(path).map_err(err)?;
+        let dt = ds.raw_datatype().map_err(err)?;
+        let maxshape = ds.max_dimensions().map_err(err)?.map(|dims| {
+            dims.into_iter()
+                .map(|d| (d != u64::MAX).then_some(d))
+                .collect()
+        });
+        Ok(DatasetInfo {
+            shape: ds.shape().map_err(err)?,
+            maxshape,
+            dtype: describe(&dt),
+            element_shape: element_shape(&dt),
+        })
+    }
+
+    /// The attributes of the group or dataset at `path`, sorted by name, and
+    /// one message per attribute that could not be read at all. An attribute
+    /// whose type has no plain JavaScript form is returned as
+    /// [`AttrValue::Raw`].
+    pub fn attrs(&self, path: &str) -> Result<(Vec<Attr>, Vec<String>)> {
+        let (map, errors) = match self.kind(path)? {
+            Kind::Dataset => self
+                .file
+                .dataset(path)
+                .and_then(|d| d.attrs_with_errors())
+                .map_err(err)?,
+            Kind::Group => self
+                .file
+                .group(path)
+                .and_then(|g| g.attrs_with_errors())
+                .map_err(err)?,
+        };
+        let mut attrs: Vec<Attr> = map
+            .into_iter()
+            .map(|(name, value)| Attr { name, value })
+            .collect();
+        attrs.sort_by(|a, b| a.name.cmp(&b.name));
+        Ok((attrs, errors.into_iter().map(err).collect()))
+    }
+
+    /// Read the dataset at `path`, whole or a hyperslab of it.
+    pub fn read(&self, path: &str, slab: Option<&Hyperslab>) -> Result<Array> {
+        let ds = self.file.dataset(path).map_err(err)?;
+        let dt = ds.raw_datatype().map_err(err)?;
+        let shape = ds.shape().map_err(err)?;
+        let (selection, mut out_shape) = match slab {
+            None => (Selection::All, shape.clone()),
+            Some(h) => hyperslab_selection(h, &shape)?,
+        };
+        let raw = ds.read_selection(&selection).map_err(err)?;
+        let data = self.decode(&raw, &dt)?;
+        out_shape.extend(element_shape(&dt));
+        let expected = out_shape
+            .iter()
+            .try_fold(1u64, |acc, &d| acc.checked_mul(d))
+            .ok_or("selection size overflows")?;
+        if data.len() as u64 != expected {
+            return Err(format!(
+                "read {} values for shape {out_shape:?} ({expected} expected)",
+                data.len()
+            ));
+        }
+        Ok(Array {
+            shape: out_shape,
+            data,
+        })
+    }
+
+    fn decode(&self, raw: &[u8], dt: &Datatype) -> Result<Data> {
+        let base = array_base(dt);
+        let is_array = !std::ptr::eq(base, dt);
+        Ok(match base {
+            Datatype::FloatingPoint { size, .. } if *size <= 4 => {
+                Data::F32(data_read::read_as_f32(raw, dt).map_err(err)?)
+            }
+            Datatype::FloatingPoint { .. } => {
+                Data::F64(data_read::read_as_f64(raw, dt).map_err(err)?)
+            }
+            Datatype::FixedPoint { size, signed, .. } => {
+                let signed_ints = || data_read::read_as_i64(raw, dt).map_err(err);
+                let unsigned_ints = || data_read::read_as_u64(raw, dt).map_err(err);
+                match (size, signed) {
+                    (1, true) => Data::I8(narrow(signed_ints()?)?),
+                    (2, true) => Data::I16(narrow(signed_ints()?)?),
+                    (4, true) => Data::I32(narrow(signed_ints()?)?),
+                    (_, true) => Data::I64(signed_ints()?),
+                    (1, false) => Data::U8(narrow(unsigned_ints()?)?),
+                    (2, false) => Data::U16(narrow(unsigned_ints()?)?),
+                    (4, false) => Data::U32(narrow(unsigned_ints()?)?),
+                    (_, false) => Data::U64(unsigned_ints()?),
+                }
+            }
+            Datatype::String { .. } if !is_array => {
+                Data::Strings(data_read::read_as_strings(raw, dt).map_err(err)?)
+            }
+            Datatype::VariableLength {
+                is_string: true, ..
+            } if !is_array => {
+                let size = dt.type_size() as usize;
+                if size == 0 || !raw.len().is_multiple_of(size) {
+                    return Err(format!(
+                        "{} bytes is not a whole number of {size}-byte string references",
+                        raw.len()
+                    ));
+                }
+                let sb = self.file.superblock();
+                Data::Strings(
+                    clawhdf5_format::vl_data::read_vl_strings(
+                        self.file.as_bytes(),
+                        raw,
+                        (raw.len() / size) as u64,
+                        sb.offset_size,
+                        sb.length_size,
+                    )
+                    .map_err(err)?,
+                )
+            }
+            Datatype::Enumeration { .. } if !is_array => {
+                Data::Strings(data_read::read_enum_names(raw, dt).map_err(err)?)
+            }
+            _ => {
+                return Err(format!(
+                    "reading {} datasets is not supported",
+                    describe(dt)
+                ));
+            }
+        })
+    }
+}
+
+/// Narrow integers read at 64 bits to the dataset's own width. The source is
+/// that width, so this cannot fail on correct input; it is checked anyway.
+fn narrow<S: Copy + std::fmt::Display, T: TryFrom<S>>(v: Vec<S>) -> Result<Vec<T>> {
+    v.into_iter()
+        .map(|x| T::try_from(x).map_err(|_| format!("value {x} out of range")))
+        .collect()
+}
+
+/// Innermost element type of (possibly nested) array datatypes.
+fn array_base(dt: &Datatype) -> &Datatype {
+    match dt {
+        Datatype::Array { base_type, .. } => array_base(base_type),
+        _ => dt,
+    }
+}
+
+fn element_shape(dt: &Datatype) -> Vec<u64> {
+    match dt {
+        Datatype::Array {
+            base_type,
+            dimensions,
+        } => {
+            let mut dims: Vec<u64> = dimensions.iter().map(|&d| u64::from(d)).collect();
+            dims.extend(element_shape(base_type));
+            dims
+        }
+        _ => Vec::new(),
+    }
+}
+
+fn hyperslab_selection(h: &Hyperslab, shape: &[u64]) -> Result<(Selection, Vec<u64>)> {
+    let rank = shape.len();
+    let ones = vec![1u64; rank];
+    let stride = h.stride.clone().unwrap_or_else(|| ones.clone());
+    let block = h.block.clone().unwrap_or(ones);
+    for (what, v) in [
+        ("start", &h.start),
+        ("count", &h.count),
+        ("stride", &stride),
+        ("block", &block),
+    ] {
+        if v.len() != rank {
+            return Err(format!(
+                "hyperslab {what} has {} dimensions, the dataset has {rank}",
+                v.len()
+            ));
+        }
+    }
+    let mut out = Vec::with_capacity(rank);
+    for d in 0..rank {
+        if stride[d] == 0 || block[d] == 0 {
+            return Err(format!("hyperslab stride and block must be >= 1 (dim {d})"));
+        }
+        if h.count[d] > 1 && block[d] > stride[d] {
+            return Err(format!(
+                "hyperslab blocks overlap in dim {d}: block {} > stride {}",
+                block[d], stride[d]
+            ));
+        }
+        // Last element selected: start + (count-1)*stride + block - 1.
+        if h.count[d] > 0 {
+            let last = (h.count[d] - 1)
+                .checked_mul(stride[d])
+                .and_then(|x| x.checked_add(h.start[d]))
+                .and_then(|x| x.checked_add(block[d] - 1));
+            match last {
+                Some(l) if l < shape[d] => {}
+                _ => {
+                    return Err(format!(
+                        "hyperslab exceeds dimension {d} (extent {})",
+                        shape[d]
+                    ));
+                }
+            }
+        }
+        out.push(
+            h.count[d]
+                .checked_mul(block[d])
+                .ok_or("selection size overflows")?,
+        );
+    }
+    Ok((
+        Selection::Hyperslab {
+            start: h.start.clone(),
+            stride,
+            count: h.count.clone(),
+            block,
+        },
+        out,
+    ))
+}
+
+/// A short, human-readable datatype name.
+pub fn describe(dt: &Datatype) -> String {
+    fn endian(order: &DatatypeByteOrder) -> &'static str {
+        match order {
+            DatatypeByteOrder::BigEndian => " (big-endian)",
+            DatatypeByteOrder::Vax => " (VAX)",
+            _ => "",
+        }
+    }
+    match dt {
+        Datatype::FixedPoint {
+            size,
+            signed,
+            byte_order,
+            ..
+        } => format!(
+            "{}{}{}",
+            if *signed { "i" } else { "u" },
+            size * 8,
+            endian(byte_order)
+        ),
+        Datatype::FloatingPoint {
+            size, byte_order, ..
+        } => format!("f{}{}", size * 8, endian(byte_order)),
+        Datatype::Time { size, .. } => format!("time{}", size * 8),
+        Datatype::String { size, .. } => format!("string[{size}]"),
+        Datatype::BitField { size, .. } => format!("bitfield{}", size * 8),
+        Datatype::Opaque { size, .. } => format!("opaque[{size}]"),
+        Datatype::Compound { members, .. } => {
+            let fields: Vec<String> = members
+                .iter()
+                .map(|m| format!("{}: {}", m.name, describe(&m.datatype)))
+                .collect();
+            format!("compound{{{}}}", fields.join(", "))
+        }
+        Datatype::Reference { .. } => "reference".to_string(),
+        Datatype::Enumeration {
+            base_type, members, ..
+        } => {
+            let names: Vec<&str> = members.iter().map(|m| m.name.as_str()).collect();
+            format!("enum<{}>{{{}}}", describe(base_type), names.join(", "))
+        }
+        Datatype::VariableLength {
+            is_string: true, ..
+        } => "vlen string".to_string(),
+        Datatype::VariableLength { base_type, .. } => {
+            format!("vlen<{}>", describe(base_type))
+        }
+        Datatype::Array {
+            base_type,
+            dimensions,
+        } => format!("array{dimensions:?}<{}>", describe(base_type)),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use clawhdf5::FileBuilder;
+
+    fn sample() -> Reader {
+        let mut b = FileBuilder::new();
+        b.create_dataset("grid")
+            .with_f64_data(&(0..12).map(f64::from).collect::<Vec<_>>())
+            .with_shape(&[3, 4])
+            .with_chunks(&[2, 2])
+            .with_deflate(4);
+        b.create_dataset("bytes").with_u8_data(&[1, 2, 250]);
+        let mut g = b.create_group("sensors");
+        g.create_dataset("temp").with_f32_data(&[1.5, -2.25]);
+        g.set_attr("location", AttrValue::String("lab".into()));
+        b.add_group(g.finish());
+        b.set_attr("version", AttrValue::I64(3));
+        b.set_attr("scale", AttrValue::F64Array(vec![0.5, 2.0]));
+        Reader::open(b.finish().unwrap()).unwrap()
+    }
+
+    #[test]
+    fn lists_groups_then_datasets() {
+        let r = sample();
+        let names: Vec<(String, Kind)> = r
+            .list("/")
+            .unwrap()
+            .into_iter()
+            .map(|c| (c.name, c.kind))
+            .collect();
+        assert_eq!(names[0], ("sensors".to_string(), Kind::Group));
+        let mut ds: Vec<&str> = names[1..].iter().map(|(n, _)| n.as_str()).collect();
+        ds.sort();
+        assert_eq!(ds, ["bytes", "grid"]);
+        assert_eq!(
+            r.list("sensors").unwrap(),
+            vec![Child {
+                name: "temp".into(),
+                kind: Kind::Dataset
+            }]
+        );
+        assert!(r.list("grid").unwrap_err().contains("not a group"));
+        assert!(r.list("missing").is_err());
+    }
+
+    #[test]
+    fn info_reports_shape_and_dtype() {
+        let r = sample();
+        let i = r.info("grid").unwrap();
+        assert_eq!(i.shape, vec![3, 4]);
+        assert_eq!(i.dtype, "f64");
+        assert!(i.element_shape.is_empty());
+        assert_eq!(r.info("sensors/temp").unwrap().dtype, "f32");
+        assert_eq!(r.kind("/sensors").unwrap(), Kind::Group);
+        assert_eq!(r.kind("/sensors/temp").unwrap(), Kind::Dataset);
+    }
+
+    #[test]
+    fn reads_whole_and_hyperslab() {
+        let r = sample();
+        let all = r.read("grid", None).unwrap();
+        assert_eq!(all.shape, vec![3, 4]);
+        assert_eq!(all.data, Data::F64((0..12).map(f64::from).collect()));
+
+        let slab = Hyperslab {
+            start: vec![1, 0],
+            count: vec![2, 2],
+            stride: Some(vec![1, 2]),
+            block: None,
+        };
+        let part = r.read("grid", Some(&slab)).unwrap();
+        assert_eq!(part.shape, vec![2, 2]);
+        assert_eq!(part.data, Data::F64(vec![4.0, 6.0, 8.0, 10.0]));
+
+        assert_eq!(
+            r.read("bytes", None).unwrap().data,
+            Data::U8(vec![1, 2, 250])
+        );
+        assert_eq!(
+            r.read("sensors/temp", None).unwrap().data,
+            Data::F32(vec![1.5, -2.25])
+        );
+    }
+
+    #[test]
+    fn bad_hyperslabs_are_refused() {
+        let r = sample();
+        let mk = |start: Vec<u64>, count: Vec<u64>| Hyperslab {
+            start,
+            count,
+            stride: None,
+            block: None,
+        };
+        assert!(
+            r.read("grid", Some(&mk(vec![0], vec![1])))
+                .unwrap_err()
+                .contains("dimensions")
+        );
+        assert!(
+            r.read("grid", Some(&mk(vec![2, 0], vec![2, 1])))
+                .unwrap_err()
+                .contains("exceeds")
+        );
+        let overlap = Hyperslab {
+            start: vec![0, 0],
+            count: vec![2, 1],
+            stride: Some(vec![1, 1]),
+            block: Some(vec![2, 1]),
+        };
+        assert!(
+            r.read("grid", Some(&overlap))
+                .unwrap_err()
+                .contains("overlap")
+        );
+    }
+
+    #[test]
+    fn attrs_are_sorted() {
+        let r = sample();
+        let (attrs, errors) = r.attrs("/").unwrap();
+        assert!(errors.is_empty());
+        let names: Vec<&str> = attrs.iter().map(|a| a.name.as_str()).collect();
+        assert_eq!(names, ["scale", "version"]);
+        let (g, _) = r.attrs("sensors").unwrap();
+        assert!(matches!(&g[0].value, AttrValue::String(s) if s == "lab"));
+    }
+
+    #[test]
+    fn compound_is_refused_not_reinterpreted() {
+        use clawhdf5::CompoundTypeBuilder;
+        let ct = CompoundTypeBuilder::new()
+            .f64_field("x")
+            .i32_field("n")
+            .build();
+        let mut rec = Vec::new();
+        rec.extend_from_slice(&1.0f64.to_le_bytes());
+        rec.extend_from_slice(&7i32.to_le_bytes());
+        let mut b = FileBuilder::new();
+        b.create_dataset("table").with_compound_data(ct, rec, 1);
+        let r = Reader::open(b.finish().unwrap()).unwrap();
+        let e = r.read("table", None).unwrap_err();
+        assert!(e.contains("compound{x: f64, n: i32}"), "{e}");
+        assert!(e.contains("not supported"), "{e}");
+    }
+
+    #[test]
+    fn garbage_is_an_error() {
+        assert!(Reader::open(vec![0u8; 64]).is_err());
+        assert!(Reader::open(Vec::new()).is_err());
+    }
+}

crates/clawhdf5-wasm/src/lib.rs

@@ -0,0 +1,244 @@
+//! clawhdf5's HDF5 reader for JavaScript, via `wasm-bindgen`.
+//!
+//! ```js
+//! import init, { open } from "./pkg/clawhdf5_wasm.js";
+//! await init();
+//! const file = open(new Uint8Array(await blob.arrayBuffer()));
+//! file.list("/");            // [{ name, kind: "group" | "dataset" }]
+//! file.info("/x");           // { shape, maxshape, dtype, elementShape }
+//! file.attrs("/x");          // [{ name, value, dtype }]
+//! file.read("/x");           // { shape, dtype, data: Float64Array | ... | string[] }
+//! file.readHyperslab("/x", [0, 0], [10, 10]);    // stride, block optional
+//! file.free();
+//! ```
+//!
+//! Numeric data comes back in the typed array of the stored width
+//! (`Int16Array` for `i16`, `BigInt64Array` for `i64`, `Float32Array` for
+//! `f32` and `f16`, ...); strings and enumeration names as arrays of strings.
+//! Anything else is a thrown `Error` naming the datatype. Only the reader is
+//! exposed: nothing here writes files.
+//!
+//! The logic lives in [`core`], which is plain Rust and tested natively.
+
+pub mod core;
+
+use clawhdf5::AttrValue;
+use js_sys::{Array, Object, Reflect};
+use wasm_bindgen::prelude::*;
+
+use crate::core::{Data, Hyperslab, Reader};
+
+/// JavaScript numbers are exact up to 2^53.
+const MAX_SAFE_INTEGER: f64 = 9_007_199_254_740_991.0;
+
+fn js_err(msg: String) -> JsError {
+    JsError::new(&msg)
+}
+
+fn set(obj: &Object, key: &str, value: impl Into<JsValue>) {
+    // Defining a property on a fresh plain object cannot fail.
+    Reflect::set(obj, &JsValue::from_str(key), &value.into()).unwrap_throw();
+}
+
+fn shape_to_js(shape: &[u64]) -> Array {
+    shape.iter().map(|&d| JsValue::from_f64(d as f64)).collect()
+}
+
+fn indices_from_js(what: &str, v: &[f64]) -> Result<Vec<u64>, JsError> {
+    v.iter()
+        .map(|&x| {
+            if x.is_finite() && x >= 0.0 && x.fract() == 0.0 && x <= MAX_SAFE_INTEGER {
+                Ok(x as u64)
+            } else {
+                Err(js_err(format!(
+                    "{what} must hold non-negative integers, got {x}"
+                )))
+            }
+        })
+        .collect()
+}
+
+fn data_to_js(data: Data) -> JsValue {
+    match data {
+        Data::F32(v) => js_sys::Float32Array::from(&v[..]).into(),
+        Data::F64(v) => js_sys::Float64Array::from(&v[..]).into(),
+        Data::I8(v) => js_sys::Int8Array::from(&v[..]).into(),
+        Data::I16(v) => js_sys::Int16Array::from(&v[..]).into(),
+        Data::I32(v) => js_sys::Int32Array::from(&v[..]).into(),
+        Data::I64(v) => js_sys::BigInt64Array::from(&v[..]).into(),
+        Data::U8(v) => js_sys::Uint8Array::from(&v[..]).into(),
+        Data::U16(v) => js_sys::Uint16Array::from(&v[..]).into(),
+        Data::U32(v) => js_sys::Uint32Array::from(&v[..]).into(),
+        Data::U64(v) => js_sys::BigUint64Array::from(&v[..]).into(),
+        Data::Strings(v) => v
+            .into_iter()
+            .map(|s| JsValue::from_str(&s))
+            .collect::<Array>()
+            .into(),
+    }
+}
+
+/// A scalar integer as a `number` when exact, else a `bigint`.
+fn int_to_js(x: i128) -> JsValue {
+    if (x as f64).abs() <= MAX_SAFE_INTEGER {
+        JsValue::from_f64(x as f64)
+    } else if let Ok(v) = i64::try_from(x) {
+        JsValue::from(v)
+    } else {
+        JsValue::from(x as u64)
+    }
+}
+
+/// An attribute value, and the datatype of one that has no JavaScript form.
+fn attr_to_js(value: AttrValue) -> (JsValue, Option<String>) {
+    match value {
+        AttrValue::F64(x) => (JsValue::from_f64(x), None),
+        AttrValue::F64Array(v) => (js_sys::Float64Array::from(&v[..]).into(), None),
+        AttrValue::I64(x) => (int_to_js(i128::from(x)), None),
+        AttrValue::I64Array(v) => (js_sys::BigInt64Array::from(&v[..]).into(), None),
+        AttrValue::U64(x) => (int_to_js(i128::from(x)), None),
+        AttrValue::U64Array(v) => (js_sys::BigUint64Array::from(&v[..]).into(), None),
+        AttrValue::String(s) => (JsValue::from_str(&s), None),
+        AttrValue::StringArray(v) => (
+            v.iter()
+                .map(|s| JsValue::from_str(s))
+                .collect::<Array>()
+                .into(),
+            None,
+        ),
+        AttrValue::Raw { datatype, .. } => (JsValue::NULL, Some(core::describe(&datatype))),
+    }
+}
+
+/// An open HDF5 (or NetCDF-4) file.
+#[wasm_bindgen]
+pub struct H5File {
+    inner: Reader,
+}
+
+/// Open a file from its bytes. Throws if they are not an HDF5 file.
+#[wasm_bindgen]
+pub fn open(bytes: Vec<u8>) -> Result<H5File, JsError> {
+    H5File::new(bytes)
+}
+
+/// The clawhdf5 version this module was built from.
+#[wasm_bindgen]
+pub fn version() -> String {
+    env!("CARGO_PKG_VERSION").to_string()
+}
+
+#[wasm_bindgen]
+impl H5File {
+    /// Same as [`open`].
+    #[wasm_bindgen(constructor)]
+    pub fn new(bytes: Vec<u8>) -> Result<H5File, JsError> {
+        Ok(H5File {
+            inner: Reader::open(bytes).map_err(js_err)?,
+        })
+    }
+
+    /// `"group"` or `"dataset"`.
+    pub fn kind(&self, path: &str) -> Result<String, JsError> {
+        Ok(self.inner.kind(path).map_err(js_err)?.as_str().to_string())
+    }
+
+    /// The group's members: `[{ name, kind }]`, groups first.
+    pub fn list(&self, path: &str) -> Result<Array, JsError> {
+        Ok(self
+            .inner
+            .list(path)
+            .map_err(js_err)?
+            .into_iter()
+            .map(|c| {
+                let o = Object::new();
+                set(&o, "name", c.name);
+                set(&o, "kind", c.kind.as_str());
+                JsValue::from(o)
+            })
+            .collect())
+    }
+
+    /// `{ shape, maxshape, dtype, elementShape }`. `maxshape` is `null`
+    /// when not recorded, with `null` for each unlimited dimension.
+    pub fn info(&self, path: &str) -> Result<Object, JsError> {
+        let i = self.inner.info(path).map_err(js_err)?;
+        let o = Object::new();
+        set(&o, "shape", shape_to_js(&i.shape));
+        let max: JsValue = match i.maxshape {
+            None => JsValue::NULL,
+            Some(dims) => dims
+                .into_iter()
+                .map(|d| d.map_or(JsValue::NULL, |d| JsValue::from_f64(d as f64)))
+                .collect::<Array>()
+                .into(),
+        };
+        set(&o, "maxshape", max);
+        set(&o, "dtype", i.dtype);
+        set(&o, "elementShape", shape_to_js(&i.element_shape));
+        Ok(o)
+    }
+
+    /// `[{ name, value, dtype }]`, sorted by name. Scalars are `number`
+    /// (`bigint` beyond 2^53) or `string`; arrays are typed arrays or
+    /// `string[]`. An attribute with no JavaScript form has `value: null`
+    /// and its `dtype`; one that could not be read at all is reported by
+    /// [`attrErrors`](Self::attr_errors).
+    pub fn attrs(&self, path: &str) -> Result<Array, JsError> {
+        let (attrs, _) = self.inner.attrs(path).map_err(js_err)?;
+        Ok(attrs
+            .into_iter()
+            .map(|a| {
+                let o = Object::new();
+                set(&o, "name", a.name);
+                let (value, dtype) = attr_to_js(a.value);
+                set(&o, "value", value);
+                set(&o, "dtype", dtype.map_or(JsValue::NULL, JsValue::from));
+                JsValue::from(o)
+            })
+            .collect())
+    }
+
+    /// Messages for attributes that could not be read.
+    #[wasm_bindgen(js_name = attrErrors)]
+    pub fn attr_errors(&self, path: &str) -> Result<Array, JsError> {
+        let (_, errors) = self.inner.attrs(path).map_err(js_err)?;
+        Ok(errors.into_iter().map(JsValue::from).collect())
+    }
+
+    /// The whole dataset: `{ shape, dtype, data }`, `data` in row-major
+    /// order.
+    pub fn read(&self, path: &str) -> Result<Object, JsError> {
+        self.read_impl(path, None)
+    }
+
+    /// A regular hyperslab (`H5Sselect_hyperslab`): `stride` and `block`
+    /// default to 1. The result's shape is `count * block` per dimension.
+    #[wasm_bindgen(js_name = readHyperslab)]
+    pub fn read_hyperslab(
+        &self,
+        path: &str,
+        start: Vec<f64>,
+        count: Vec<f64>,
+        stride: Option<Vec<f64>>,
+        block: Option<Vec<f64>>,
+    ) -> Result<Object, JsError> {
+        let slab = Hyperslab {
+            start: indices_from_js("start", &start)?,
+            count: indices_from_js("count", &count)?,
+            stride: stride.map(|s| indices_from_js("stride", &s)).transpose()?,
+            block: block.map(|b| indices_from_js("block", &b)).transpose()?,
+        };
+        self.read_impl(path, Some(&slab))
+    }
+
+    fn read_impl(&self, path: &str, slab: Option<&Hyperslab>) -> Result<Object, JsError> {
+        let dtype = self.inner.info(path).map_err(js_err)?.dtype;
+        let a = self.inner.read(path, slab).map_err(js_err)?;
+        let o = Object::new();
+        set(&o, "shape", shape_to_js(&a.shape));
+        set(&o, "dtype", dtype);
+        set(&o, "data", data_to_js(a.data));
+        Ok(o)
+    }
+}

crates/clawhdf5-wasm/tests/h5py_interop.rs

@@ -0,0 +1,265 @@
+//! The wasm reader's core against files written by h5py and netCDF4, with
+//! the values libhdf5 reads back as the reference. The generator,
+//! `examples/wasm-viewer/test/make_fixture.py`, is shared with the Node test
+//! of the built wasm package, so both compare against the same expectations.
+//!
+//! Skipped when python3 with h5py/netCDF4 is missing, unless
+//! `CLAWHDF5_REQUIRE_INTEROP=1`. `CLAWHDF5_PYTHON` names the interpreter.
+
+use std::path::{Path, PathBuf};
+use std::process::Command;
+
+use clawhdf5::AttrValue;
+use clawhdf5_wasm::core::{Data, Hyperslab, Kind, Reader};
+use serde_json::Value;
+
+fn python() -> String {
+    std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
+}
+
+fn interop_required() -> bool {
+    std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
+}
+
+fn python_available() -> bool {
+    Command::new(python())
+        .args(["-c", "import h5py, netCDF4, numpy"])
+        .output()
+        .map(|o| o.status.success())
+        .unwrap_or(false)
+}
+
+fn generator() -> PathBuf {
+    Path::new(env!("CARGO_MANIFEST_DIR")).join("../../examples/wasm-viewer/test/make_fixture.py")
+}
+
+/// Values as comparable strings: integers exactly, floats by their f64
+/// value (an f32 widens exactly), strings as themselves.
+fn data_strings(d: &Data) -> (&'static str, Vec<String>) {
+    fn s<T: ToString>(v: &[T]) -> Vec<String> {
+        v.iter().map(ToString::to_string).collect()
+    }
+    fn f<T: Copy + Into<f64>>(v: &[T]) -> Vec<String> {
+        v.iter().map(|&x| format!("{:?}", x.into())).collect()
+    }
+    match d {
+        Data::F32(v) => ("f32", f(v)),
+        Data::F64(v) => ("f64", f(v)),
+        Data::I8(v) => ("i8", s(v)),
+        Data::I16(v) => ("i16", s(v)),
+        Data::I32(v) => ("i32", s(v)),
+        Data::I64(v) => ("i64", s(v)),
+        Data::U8(v) => ("u8", s(v)),
+        Data::U16(v) => ("u16", s(v)),
+        Data::U32(v) => ("u32", s(v)),
+        Data::U64(v) => ("u64", s(v)),
+        Data::Strings(v) => ("strings", v.clone()),
+    }
+}
+
+fn expected_strings(kind: &str, values: &Value) -> Vec<String> {
+    values
+        .as_array()
+        .unwrap()
+        .iter()
+        .map(|v| match (kind, v) {
+            ("f32" | "f64", Value::Number(n)) => format!("{:?}", n.as_f64().unwrap()),
+            (_, Value::String(s)) => s.clone(),
+            other => panic!("unexpected expected value {other:?}"),
+        })
+        .collect()
+}
+
+fn shape(v: &Value) -> Vec<u64> {
+    v.as_array()
+        .unwrap()
+        .iter()
+        .map(|x| x.as_u64().unwrap())
+        .collect()
+}
+
+fn check_attr(file: &str, path: &str, name: &str, got: &AttrValue, want: &Value) {
+    let ctx = format!("{file}:{path}@{name}");
+    let ints = |v: &Value| -> Vec<String> {
+        v.as_array()
+            .unwrap()
+            .iter()
+            .map(|x| x.as_str().unwrap().to_string())
+            .collect()
+    };
+    let scalar = want["scalar"].as_bool().unwrap_or(false);
+    match got {
+        AttrValue::String(s) => assert_eq!(want["string"].as_str(), Some(s.as_str()), "{ctx}"),
+        AttrValue::StringArray(v) => {
+            let w: Vec<&str> = want["strings"]
+                .as_array()
+                .unwrap_or_else(|| panic!("{ctx}: got {v:?}"))
+                .iter()
+                .map(|x| x.as_str().unwrap())
+                .collect();
+            assert_eq!(v, &w, "{ctx}");
+        }
+        AttrValue::I64(x) => {
+            assert!(scalar, "{ctx}");
+            assert_eq!(vec![x.to_string()], ints(&want["int"]), "{ctx}");
+        }
+        AttrValue::U64(x) => {
+            assert!(scalar, "{ctx}");
+            assert_eq!(vec![x.to_string()], ints(&want["int"]), "{ctx}");
+        }
+        AttrValue::I64Array(v) => assert_eq!(
+            v.iter().map(ToString::to_string).collect::<Vec<_>>(),
+            ints(&want["int"]),
+            "{ctx}"
+        ),
+        AttrValue::U64Array(v) => assert_eq!(
+            v.iter().map(ToString::to_string).collect::<Vec<_>>(),
+            ints(&want["int"]),
+            "{ctx}"
+        ),
+        AttrValue::F64(x) => {
+            assert!(scalar, "{ctx}");
+            assert_eq!(Some(*x), want["float"][0].as_f64(), "{ctx}");
+        }
+        AttrValue::F64Array(v) => {
+            let w: Vec<f64> = want["float"]
+                .as_array()
+                .unwrap()
+                .iter()
+                .map(|x| x.as_f64().unwrap())
+                .collect();
+            assert_eq!(v, &w, "{ctx}");
+        }
+        AttrValue::Raw { datatype, .. } => {
+            let want = want["raw"]
+                .as_str()
+                .unwrap_or_else(|| panic!("{ctx}: undecoded {datatype:?}"));
+            let d = clawhdf5_wasm::core::describe(datatype);
+            assert!(d.contains(want), "{ctx}: {d}");
+        }
+    }
+}
+
+fn check_file(dir: &Path, file: &str, exp: &Value) {
+    let r = Reader::open(std::fs::read(dir.join(file)).unwrap()).unwrap();
+
+    for (path, want) in exp["lists"].as_object().unwrap() {
+        let list = r
+            .list(path)
+            .unwrap_or_else(|e| panic!("{file}:{path}: {e}"));
+        for (kind, key) in [(Kind::Group, "groups"), (Kind::Dataset, "datasets")] {
+            let mut got: Vec<&str> = list
+                .iter()
+                .filter(|c| c.kind == kind)
+                .map(|c| c.name.as_str())
+                .collect();
+            got.sort();
+            let w: Vec<&str> = want[key]
+                .as_array()
+                .unwrap()
+                .iter()
+                .map(|x| x.as_str().unwrap())
+                .collect();
+            assert_eq!(got, w, "{file}:{path} {key}");
+        }
+    }
+
+    for (path, want) in exp["datasets"].as_object().unwrap() {
+        let kind = want["kind"].as_str().unwrap();
+        let a = match (r.read(path, None), want["unavailable"].as_str()) {
+            (Ok(a), _) => a,
+            // A filter this build may lack (zstd: the wasm build has it
+            // off, a workspace build may unify it on) must fail clearly.
+            (Err(e), Some(why)) => {
+                assert!(e.contains(why), "{file}:{path}: {e}");
+                continue;
+            }
+            (Err(e), None) => panic!("{file}:{path}: {e}"),
+        };
+        assert_eq!(a.shape, shape(&want["shape"]), "{file}:{path} shape");
+        let (got_kind, got) = data_strings(&a.data);
+        assert_eq!(got_kind, kind, "{file}:{path} kind");
+        assert_eq!(
+            got,
+            expected_strings(kind, &want["values"]),
+            "{file}:{path}"
+        );
+
+        if let Some(slab) = want.get("slab") {
+            let h = Hyperslab {
+                start: shape(&slab["start"]),
+                count: shape(&slab["count"]),
+                stride: Some(shape(&slab["stride"])),
+                block: None,
+            };
+            let a = r
+                .read(path, Some(&h))
+                .unwrap_or_else(|e| panic!("{file}:{path} {h:?}: {e}"));
+            assert_eq!(a.shape, shape(&slab["shape"]), "{file}:{path} slab shape");
+            assert_eq!(
+                data_strings(&a.data).1,
+                expected_strings(kind, &slab["values"]),
+                "{file}:{path} slab"
+            );
+        }
+    }
+
+    for (path, what) in exp["errors"].as_object().unwrap() {
+        let e = r.read(path, None).expect_err(path);
+        assert!(e.contains(what.as_str().unwrap()), "{file}:{path}: {e}");
+    }
+
+    let skip: Vec<&str> = exp["skip_attrs"]
+        .as_array()
+        .unwrap()
+        .iter()
+        .map(|x| x.as_str().unwrap())
+        .collect();
+    for (path, want) in exp["attrs"].as_object().unwrap() {
+        let (attrs, errors) = r
+            .attrs(path)
+            .unwrap_or_else(|e| panic!("{file}:{path}: {e}"));
+        assert!(errors.is_empty(), "{file}:{path}: {errors:?}");
+        let want = want.as_object().unwrap();
+        let mut compared = 0;
+        for a in &attrs {
+            if a.name.starts_with('_') || skip.contains(&a.name.as_str()) {
+                continue;
+            }
+            let w = want
+                .get(&a.name)
+                .unwrap_or_else(|| panic!("{file}:{path}: unexpected attribute {}", a.name));
+            check_attr(file, path, &a.name, &a.value, w);
+            compared += 1;
+        }
+        assert_eq!(compared, want.len(), "{file}:{path}: attributes missing");
+    }
+}
+
+#[test]
+fn reads_what_h5py_and_netcdf4_wrote() {
+    if !python_available() {
+        assert!(
+            !interop_required(),
+            "CLAWHDF5_REQUIRE_INTEROP=1 but python with h5py, netCDF4 and numpy is not available"
+        );
+        eprintln!("SKIP: python with h5py/netCDF4 not available");
+        return;
+    }
+    let dir = tempfile::tempdir().unwrap();
+    let out = Command::new(python())
+        .arg(generator())
+        .arg(dir.path())
+        .output()
+        .expect("run python");
+    assert!(
+        out.status.success(),
+        "fixture generator failed:\n{}",
+        String::from_utf8_lossy(&out.stderr)
+    );
+    let exp: Value =
+        serde_json::from_slice(&std::fs::read(dir.path().join("expected.json")).unwrap()).unwrap();
+    for file in ["fixture.h5", "fixture.nc"] {
+        check_file(dir.path(), file, &exp[file]);
+    }
+}

crates/clawhdf5/Cargo.toml

@@ -31,7 +31,7 @@ name = "parallel_bench"
 harness = false
 
 [features]
-default = ["mmap", "provenance"]
+default = ["mmap", "provenance", "lzf"]
 mmap = ["clawhdf5-io/mmap"]
 parallel = ["clawhdf5-format/parallel", "rayon"]
 # zlib-ng (C, needs cmake) instead of the default pure-Rust zlib-rs.
@@ -41,6 +41,14 @@ zstd = ["clawhdf5-format/zstd"]
 blake3_hash = ["clawhdf5-format/blake3_hash"]
 lz4 = ["clawhdf5-format/lz4"]
 pcodec = ["clawhdf5-format/pcodec"]
+# Plugin filters, pure Rust (no C). LZF (32000) is h5py's built-in
+# compression; it has no dependencies, so it is on by default.
+lzf = ["clawhdf5-format/lzf"]
+bitshuffle = ["clawhdf5-format/bitshuffle"]
+bzip2 = ["clawhdf5-format/bzip2"]
+blosc = ["clawhdf5-format/blosc"]
+# Every plugin filter.
+plugin-filters = ["lzf", "bitshuffle", "bzip2", "blosc"]
 # Dataset::verify_provenance() — recompute a dataset's SHA-256 and compare
 # against its stored _provenance_sha256 attribute. On by default, matching
 # clawhdf5-format's own default-on `provenance` feature.