docs: record the 2026-09-25 HDF5 audit fixes and open gaps

CHANGELOG: upgrade notes (changed read results for max-shape files,
saturating conversions, new writer errors, format-crate API changes) and
the reader/writer correctness fixes. known-issues: the silent-wrong-data
table with before/after sweep numbers, the gaps still open, and a
correction to the Extensible Array entry, which said files we wrote were
unaffected. CLAUDE.md: clawhdf5-gpu is vector distance computation, not
I/O, and clawhdf5-filters holds only deflate backends (no Blosc).

Also a facade test that libhdf5's 20-bit N-Bit float test data reads as
libhdf5's values.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>

.gitea/workflows/ci.yml

@@ -9,15 +9,15 @@ jobs:
     runs-on: ubuntu-latest
     container: rust:latest
     steps:
-      - uses: actions/checkout@v4
-      - name: Cache cargo registry/target
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cargo/registry
-            ~/.cargo/git
-            target
-          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+      # Plain git rather than actions/checkout: that is a JavaScript action,
+      # and rust:latest has no `node`, so it failed with exit 127 before any
+      # code was built — on every push. actions/cache went for the same reason.
+      - name: Check out
+        run: |
+          git init -q .
+          git remote add origin "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git"
+          for i in 1 2 3; do git fetch -q --depth 1 origin "${GITHUB_SHA}" && break; sleep 5; done
+          git checkout -q FETCH_HEAD
       - name: Install rustfmt & clippy components
         run: rustup component add rustfmt clippy
       - name: Install thumbv7em-none-eabihf target
@@ -28,12 +28,15 @@ jobs:
         # dependency a failure (CLAWHDF5_REQUIRE_INTEROP below).
         run: |
           apt-get update
-          apt-get install -y --no-install-recommends python3 python3-venv
+          # cmake builds libz-ng-sys for the opt-in `fast-deflate` (zlib-ng)
+          # steps in ci-test.sh; rust:latest does not ship it. The default
+          # build (pure-Rust zlib-rs) does not need it.
+          apt-get install -y --no-install-recommends python3 python3-venv cmake
           python3 -m venv /opt/interop
-          /opt/interop/bin/pip install --no-cache-dir h5py numpy netCDF4 xarray
+          /opt/interop/bin/pip install --no-cache-dir h5py numpy netCDF4 xarray hdf5plugin
           echo "/opt/interop/bin" >> "$GITHUB_PATH"
       - name: Show interop library versions
-        run: /opt/interop/bin/python -c "import h5py, netCDF4; print('h5py', h5py.__version__, 'HDF5', h5py.version.hdf5_version, 'netCDF4', netCDF4.__version__)"
+        run: /opt/interop/bin/python -c "import h5py, netCDF4, hdf5plugin; print('h5py', h5py.__version__, 'HDF5', h5py.version.hdf5_version, 'netCDF4', netCDF4.__version__, 'hdf5plugin', hdf5plugin.version)"
       - name: Run CI script
         env:
           # Name the interpreter outright rather than relying on $GITHUB_PATH
@@ -44,3 +47,40 @@ jobs:
           CLAWHDF5_PYTHON: /opt/interop/bin/python
           CLAWHDF5_REQUIRE_INTEROP: "1"
         run: bash scripts/ci-test.sh
+
+  test-arm64:
+    # The aarch64 kernels in clawhdf5-accel — NEON `dot_i8`, including the
+    # SDOT path, and the f32 NEON kernels — are cfg'd out on x86, so the job
+    # above never compiles, lints or tests them.
+    #
+    # `linux_arm64` is served by two runners that execute differently:
+    # vision-01 runs steps on the host (Rust already installed) and vision-02
+    # runs them in docker.gitea.com/runner-images. So the steps work in both:
+    # no `container:`, no JavaScript actions (they are fetched from GitHub,
+    # which not every runner reliably reaches), and an explicit `+stable`
+    # toolchain rather than whatever a host happens to default to.
+    runs-on: linux_arm64
+    env:
+      CARGO_NET_RETRY: "10"
+      CARGO_TERM_COLOR: always
+    steps:
+      - name: Check out
+        run: |
+          git init -q .
+          git remote add origin "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git"
+          for i in 1 2 3; do git fetch -q --depth 1 origin "${GITHUB_SHA}" && break; sleep 5; done
+          git checkout -q FETCH_HEAD
+      - name: Rust stable
+        run: |
+          export PATH="$HOME/.cargo/bin:$PATH"
+          command -v rustup >/dev/null || curl -sSf --retry 5 https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain none
+          rustup toolchain install stable --profile minimal --component clippy
+          echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
+      - name: Confirm aarch64
+        run: |
+          test "$(uname -m)" = aarch64
+          if grep -q asimddp /proc/cpuinfo; then echo "dot-product extension present: SDOT kernel runs"; else echo "no dot-product extension: plain NEON kernel runs"; fi
+      - name: Clippy (aarch64 kernels)
+        run: cargo +stable clippy -p clawhdf5-accel --all-targets -- -D warnings
+      - name: Test
+        run: cargo +stable test -p clawhdf5-accel -p clawhdf5-ann -p clawhdf5-format

CHANGELOG.md

@@ -1,5 +1,395 @@
 # Changelog
 
+## Unreleased
+
+### Upgrade Notes
+- **HDF5 correctness audit (2026-09-25).** A sweep of 686 public files (the
+  libhdf5 test files, the HDF Group's CVE reproducers, pyfive, netcdf-c,
+  netcdf4-python, h5wasm, h5py and xarray corpora), a 567-case read matrix and
+  a 96-case write matrix against HDF5 1.10–2.0 found bugs that returned wrong
+  values with no error, and files we wrote that libhdf5 rejects. The fixes are
+  listed under Correctness and Interop. What changes for callers:
+  - **Chunked datasets whose max shape is larger than their current shape**,
+    or whose unlimited dimension is not the first, were indexed by the current
+    shape instead of the max shape, both when read and when written. Files from
+    libhdf5 now read correctly. Files clawhdf5 wrote with such a max shape were
+    laid out wrongly and now read the way libhdf5 always read them — rewrite
+    them. Agent stores and ClawBrainHub files have no max shape and are
+    unaffected.
+  - Integer reads (`read_i32`/`read_i64`/`read_u64`/...) of float data now
+    convert (truncate toward zero, saturate at the type's range, NaN reads as
+    0) instead of returning the IEEE bit pattern, and out-of-range integers
+    saturate instead of keeping the low bits.
+  - `FileWriter::finish()` now returns an error instead of writing a corrupt
+    file for: a header message over 64 KiB (e.g. an attribute larger than
+    ~64 KiB), a group/dataset/link name that is empty, `.` or contains `/`
+    (nested paths were written as one literal link), a max shape smaller than
+    the shape, a page size outside 512 B–1 GiB, and more than 65 535 chunks in
+    a dataset with several unlimited dimensions.
+  - **Breaking (format crate):** `ObjectHeaderWriter::serialize`,
+    `BatchObjectHeaderWriter::compute_sizes`/`serialize_all` and
+    `build_chunked_data_from_precompressed` return `Result`;
+    `read_fixed_array_chunks`/`read_extensible_array_chunks` take `max_dims`;
+    `build_fixed_array_at`/`ea_writer::build_extensible_array_at` take one
+    `Option<WrittenChunk>` per index slot; `fill_value::dataset_fill_value`
+    returns `UnresolvedSharedMessage` for a shared message it cannot resolve
+    instead of `None`. `FillTime::default()` is `IfSet` (libhdf5's default;
+    default files are byte-identical).
+- **ZeroClaw does not use clawhdf5.** The project described itself as
+  ZeroClaw's memory backend ("imported as a `clawhdf5` Cargo feature"). Checked
+  against ZeroClaw v0.8.5 (the latest release), the `osobh/zeroclaw` fork and
+  their full history: no such feature or backend has ever existed. And
+  `clawhdf5-migrate`'s "ZeroClaw layout" (`memory_chunks`, `sessions`,
+  `entities`, `relations`) is not ZeroClaw's schema — ZeroClaw uses a single
+  `memories` table — so the migrator cannot read a ZeroClaw database. The
+  claims are withdrawn; the migrator's layout is documented as its own.
+- **OpenClaw is not supported, and never was.** The docs described a
+  "drop-in" OpenClaw memory backend enabled with `memory.backend = "clawhdf5"`.
+  That config was never valid in any OpenClaw release (v2026.2–v2026.7
+  accepted only `builtin`/`qmd` and rejected unknown keys, so a Gateway given
+  it refuses to start; OpenClaw 2.0 removed the key), no plugin was ever built,
+  and `@redclaw/clawhdf5` was never published. The integration docs
+  (`openclaw-integration.md`, `openclaw-config.md`, `migration-guide.md`) are
+  removed; `docs/openclaw.md` explains the status and what a real plugin would
+  need against OpenClaw v2026.9.6. `ClawhdfBackend` stays as a library API.
+- **Breaking:** `MemoryError` is now `#[non_exhaustive]` and gained
+  `SigningKeyRequired`; a `match` on it needs a wildcard arm. Future variants
+  will no longer be breaking.
+- **Breaking:** `clawhdf5-agent`'s `agent` feature is removed. It enabled
+  nothing — the agent layer is always built — but the README and guides told
+  people to pass it; drop `agent` from `features = [...]`.
+- **`clawhdf5-migrate` now writes a real agent store.** Its output used to be
+  a layout of its own (`/chunks`, `/sessions`, `/entities`, `/relations`, no
+  `/meta`) that `HDF5Memory::open` rejected, so a migrated file could not be
+  used as agent memory. Files it wrote before this release are not agent
+  stores; re-run the migration. Also: embeddings default to `float16` like
+  any new store (`--f32` opts out; `--float16` is a hidden no-op); a row with
+  the wrong embedding length is an error instead of being truncated or
+  padded; `--incremental` now matches rows by content against an existing
+  store and follows the source's deleted flags; a source with no memory rows
+  needs `--embedding-dim`. The per-dataset SHA-256 provenance attributes of
+  the old layout are gone (the agent schema has no place for them).
+- **Files written by clawhdf5 now open in h5py and libhdf5.** Every `f32`
+  dataset we wrote — including every agent store's embeddings — was refused
+  with "sign bit position out of bounds", and every empty dataset with
+  "invalid dataset size". Both were write-side bugs present in every release;
+  clawhdf5's own reader was unaffected. An agent store is rewritten in full at
+  each checkpoint, so it becomes readable at its next checkpoint on this
+  version; other files with `f32` or empty datasets need rewriting. Details in
+  `docs/known-issues.md`.
+- **New stores store embeddings as half precision by default.**
+  `MemoryConfig::float16` was persisted and otherwise ignored; it now writes
+  `float16` embeddings (48% smaller files at 100K) and rounds each embedding
+  to half precision as it is saved — and it defaults to `true` for new
+  stores. On the full LongMemEval haystack with real MiniLM embeddings every
+  retrieval metric matched `f32`. **Existing stores are unaffected**: every
+  agent store has recorded `float16 = false`, and keeps it (a v2.5.0 fixture
+  guards this). A store that already had `float16 = true` rounds its
+  embeddings when next opened and writes them as `float16` at its next
+  checkpoint. Opt out with `float16 = false` or `create --f32`; the CLI's
+  `--float16` is still accepted and now a no-op. Values beyond ±65504 are
+  refused, so keep `f32` for unnormalised vectors.
+- **Breaking:** `MemoryError` gained `InvalidEntry`, returned when a
+  `float16` store is given an embedding value beyond ±65504. Exhaustive
+  matches need the new arm.
+- **The default build no longer compiles any C.** Deflate now defaults to the
+  pure-Rust zlib-rs instead of zlib-ng, so building the core crates needs
+  neither cmake nor a C compiler. Speed on HDF5 reads and writes is within 6%
+  of zlib-ng, and compressed output is byte-identical. To keep zlib-ng, enable
+  `fast-deflate` (on `clawhdf5`, `clawhdf5-format` or `clawhdf5-filters`); it
+  overrides zlib-rs wherever it is on.
+- **A truncated deflate chunk is now an error.** It used to read back short,
+  with no error.
+- **Minimum supported Rust is 1.92**, now declared in every crate's
+  `rust-version` and checked in CI.
+- **New stores use the int8 vector index by default.**
+  `MemoryConfig::quantized_index` now defaults to `true`: a quarter of the
+  index memory, builds 1.8x (x86-64) and 2.3x (Raspberry Pi 5) faster, and
+  searches 1.63x and 1.18x faster at equal recall, measured on every
+  configuration tested. **Existing stores are unaffected** — a store written
+  with v2.6.0 or later keeps its persisted setting, and one written before the
+  setting existed opens as `false` and keeps its f32 index. Set
+  `quantized_index = false`, or pass `create --f32-index` to the CLI, to opt
+  out. The CLI's `--quantized-index` is still accepted but is now a no-op.
+
+### Signing
+- `clawhdf5-agent`: **Ed25519-signed checkpoints** — the README's
+  "cryptographically verifiable memory", now true. With
+  `HDF5Memory::set_signing_key(key)`, every checkpoint stores a signed
+  manifest: a SHA-256 per record (text, embedding as stored, channel,
+  timestamp, session, tags, deleted flag, activation) in a Merkle tree, plus
+  hashes of the settings (and WAL mark), sessions and knowledge graph, with
+  the per-record hashes in `/integrity/record_hashes`.
+  `HDF5Memory::verify(path, &public_key)` recomputes everything from the file
+  and reports which part changed and which records (`changed_records`); a
+  forged manifest fails the signature. The key is never persisted; a signed
+  store refuses to checkpoint without it (`MemoryError::SigningKeyRequired`),
+  and `remove_signature()` is the deliberate way back to unsigned. Saves still
+  in the WAL are not covered (`wal_entries_unsigned`). Tests include every
+  kind of edit, and an edit made with h5py in place, which verify pinpoints.
+  Cost: ~20% of a checkpoint, 32 bytes per record (`BENCHMARKS.md`, "Signed
+  checkpoints"). New dependencies `ed25519-dalek`, `sha2`, `rand_core` — pure
+  Rust; the no-C check still passes.
+- `clawhdf5-cli`: `keygen --out <file>` (owner-only key file),
+  `--signing-key <file>` / `CLAWHDF5_SIGNING_KEY` on writing commands
+  (`create` signs immediately), `verify --public-key <hex|file>` (JSON report;
+  exit status 2 if not valid), and `signed` in `create`/`stats` output.
+
+### Migration
+- `clawhdf5-migrate`: writes through the agent's own API (`HDF5Memory::create`
+  / `open`, `save_batch`, the session cache and knowledge graph), so there is
+  no second copy of the schema. Sessions and entities/relations carry over;
+  deleted rows become deleted records (or are left out with
+  `--skip-deleted`). Every source row is checked before the output is created,
+  so a source that cannot be migrated leaves an existing store untouched.
+  Validation reads the result back with `HDF5Memory::open_read_only`, compares
+  every field (embeddings bit for bit — `round_to_f16` of the source for a
+  `float16` store) and checks that a migrated record is found by search. The
+  `half`-based conversion is gone; `clawhdf5_format::float16` is the only one.
+  42 tests, including h5py opening a migrated store; an adversarial review's
+  two blocker and four major findings are fixed with regression tests.
+- `clawhdf5-agent`: `HDF5Memory::sessions()` / `sessions_mut()`,
+  `HDF5Memory::delete_batch(&[usize])` (one save, all-or-nothing, never
+  auto-compacts), `SessionCache::add_at`, and `SessionCache` / `SessionEntry`
+  re-exported from the crate root.
+
+### Search
+- `clawhdf5-agent`: **`HDF5Memory::search` with `SearchOptions`** — source
+  filtering, re-ranking and confidence rejection in the store's own search
+  path. Re-ranking and confidence rejection used to be reachable only
+  through the OpenClaw backend, which now calls `search` with both on.
+  - `with_sources([..])` restricts a search to records from those source
+    channels. It applies before ranking, so a filtered search still returns up
+    to `k` results, normalised over what it can return. Measured at 100K: the
+    exact filtered top 10 for filters keeping 50%, 10% and 1% of the store and
+    for records far from the query, and never slower than an unfiltered search
+    (2.3 ms for a 1% filter vs 4.6 ms unfiltered). See `BENCHMARKS.md`,
+    "Search options".
+  - `with_rerank(ReRankConfig)` re-ranks a pool of `max(3k, 10)` candidates
+    (`rerank_pool` to change it) by relevance, recency, source authority and
+    activation; `with_confidence(ConfidenceConfig)` drops low-confidence
+    results; `at_time(now)` pins the clock for recency. About 3% on latency.
+  - `hybrid_search` and `hybrid_search_with` are unchanged (tested bit for
+    bit against `search` with default options).
+- `clawhdf5-agent`: the OpenClaw backend's search now boosts the Hebbian
+  activation of the `k` results it returns, not of the whole `3k` candidate
+  pool it re-ranks.
+
+### Documentation
+- OpenClaw claims withdrawn across the README, QUICKSTART, USE_CASES, ROADMAP
+  (Track 7 marked withdrawn) and the `openclaw` module docs; the dead
+  `github.com/redclawsystems/openclaw` link is gone. The Node package is
+  marked unpublished and broken (now `"private": true` so it cannot be
+  published by accident), with its bugs recorded in `docs/known-issues.md`.
+
+### Benchmarks
+- Every undated or pre-September section of `BENCHMARKS.md` re-run on one
+  machine on one day (tank, 2026-09-24, commit 5c8323c), with the command for
+  each and every number traced back to the raw output by a separate check.
+  Where a figure moved, the section says so. Two apparent regressions were
+  isolated rather than published: knowledge-graph traversal (a real bug,
+  fixed above) and the write path, which measures the same at v2.3.0 on this
+  machine — the old 18 µs / 6.17 ms figures came from an undated run on other
+  hardware; `float16` adds ~2 µs per save and the int8 index nothing.
+- New `multimodal_bench`: cross-modal search at 1K and 10K records, which the
+  README claimed but nothing measured.
+- `footprint_bench` reports whether it built `float16` or `f32` stores and
+  takes `--f32`; it had kept printing "f32" after the default changed.
+
+### Interop
+- `clawhdf5-format`: **every `f32` dataset was unreadable by h5py and
+  libhdf5.** The float datatype encoder hard-coded the sign bit's position to
+  63, correct only for `f64`; libhdf5 validates it and refused the dataset. It
+  is now derived from the type (15 / 31 / 63). Our reader ignores the field,
+  and the interop suites only wrote `f64`, which is how it went unnoticed.
+- `clawhdf5-format`: **every empty dataset was unreadable by h5py and
+  libhdf5.** It was written with a real address and zero bytes, which trips
+  libhdf5's `addr + size <= addr` overflow check. An empty contiguous dataset
+  now gets the undefined address, as libhdf5 writes it. This affected every
+  agent store without sessions or a knowledge graph.
+- New interop tests: `f32` and `float16` datasets in both directions (our
+  `float16` rounding matches numpy's bit for bit on 4 020 probe values,
+  including ties, subnormals and the overflow boundary), and an agent store —
+  `f32` and `float16` — opened by h5py with every dataset decoded.
+- `clawhdf5-format` filters, checked against libhdf5 + hdf5plugin:
+  - **LZ4 (32004) now uses the registered HDF5 LZ4 format** (8-byte BE size,
+    4-byte BE block size, BE-length-prefixed blocks). Our old framing (4-byte
+    LE size + one block) was readable only by clawhdf5, and we could not read
+    libhdf5's (`h5ex_d_lz4.h5`). Old clawhdf5 LZ4 chunks still read; they are
+    told apart unambiguously (a registered chunk starts with four zero bytes).
+  - **Zstd (32015) frames now record the content size**, which libhdf5's zstd
+    plugin needs; h5py could not read our zstd datasets.
+  - **Pcodec moved from filter ID 32023 to 480.** 32023 is registered to
+    Granular BitRound, whose decode is a pass-through — libhdf5 with that
+    plugin would have returned compressed bytes as data. Pcodec has no
+    registered ID; 480 is in the registry's private range (256–511) and only
+    clawhdf5 can read it. Chunks written under 32023 with the filter name
+    `pcodec` (clawhdf5 ≤ 2.7.0) still read.
+  - **SZIP decode matches libhdf5.** It returned garbage or zeros with no
+    error for libhdf5-written files (the 4-byte size prefix, 32/64-bit
+    byte-plane interleaving, reference interval, scanline padding and byte
+    order were all handled wrongly) and rejected 64-bit data.
+  - N-Bit honours libhdf5's "need not compress" flag (multi-filter pipelines
+    such as `tfilters.h5` failed) and reads enum/no-op members.
+  - Scale-offset `float` decode uses libhdf5's single-precision arithmetic
+    (was 1 ULP off for some values).
+  - A pipeline with Fletcher32 ahead of the compressor (h5py
+    `set_fletcher32()` then `set_deflate()`) no longer fails with "deflate:
+    output exceeds size limit".
+
+### Storage
+- `clawhdf5-format`: **half-precision datasets.**
+  `DatasetBuilder::with_f16_data` writes IEEE binary16 (numpy `float16`),
+  rounding to nearest-even; `make_f16_type`, and `clawhdf5_format::float16`
+  with the conversions, which are checked against the `half` crate on 16.7M
+  values and round-trip all 65 536 half values. Reading `float16` as `f32`
+  gained a little-endian fast path.
+- `clawhdf5-agent`: **`MemoryConfig::float16` stores embeddings as half
+  precision.** At 100K x 384 the file goes from 154.0 to 80.8 MiB (−48%), a
+  checkpoint from 752 to 512 ms and open from 300 to 252 ms, with the same
+  vector recall@10 against an exact scan (0.999 vs 0.994) and the same
+  `hybrid_search` latency; at 10K open is 3 ms slower. On the full
+  LongMemEval haystack with real MiniLM embeddings every retrieval metric is
+  identical to `f32` (`longmemeval_bench --float16`). The cache rounds each
+  embedding as it is saved, so memory and file agree bit for bit and a store
+  returns the same results before and after a reopen (tested). Out-of-range
+  values are refused with `MemoryError::InvalidEntry` rather than stored as
+  infinity; batches are all or nothing. CLI: `create --float16`. See
+  `BENCHMARKS.md`, "float16 embedding storage".
+
+### Build
+- **Pure-Rust default.** `clawhdf5-format`, `clawhdf5-filters` and the
+  `clawhdf5` facade default to the `zlib-rs` deflate backend; `fast-deflate`
+  (zlib-ng) is opt-in. No crate in the default dependency tree of the core
+  crates compiles C, and `ci-test.sh` now fails if one appears. The facade's
+  `fast-deflate` was on by default and is now off. See `BENCHMARKS.md`,
+  "Deflate backend".
+- `zlib-rs` also enables flate2's `runtime_detection`. Without it zlib-rs has
+  no `std`, cannot detect SIMD at runtime, and inflates 3.5x slower; the
+  workspace builds flate2 with `default-features = false`, which had been
+  switching it off.
+- `rust-version = "1.92"` for the whole workspace (the floor: `wgpu` requires
+  it), and CI checks the workspace on exactly that toolchain.
+- CI keeps zlib-ng building and tested; the arm64 job no longer needs cmake.
+
+### Correctness
+- `clawhdf5-format` reader — **values returned wrong with no error:**
+  - Fixed Array and Extensible Array chunk indexes were laid out by the
+    dataset's current shape instead of its max shape (23 libhdf5 test files,
+    and any h5py file with e.g. `maxshape=(10, None)` or `(20, 10)` under
+    `libver='latest'`).
+  - Files with 4-byte offsets: unfiltered chunked datasets read as zeros.
+    Chunk B-tree keys store offsets in 8 bytes whatever the file's offset
+    size.
+  - A chunk's filter mask skipped the whole pipeline when any bit was set;
+    only the flagged filters are skipped now.
+  - Float data read as an integer returned the bit pattern; narrowing integer
+    reads kept the low bits; bfloat16 was decoded as IEEE half. Floats are now
+    decoded from their datatype fields (bf16, FP8 E4M3/E5M2, IEEE half, single
+    and double).
+  - `vl_data::read_vl_bytes` truncated sequences of non-byte base types.
+  - A shared fill-value message read as zero fill; it is resolved now,
+    including from the file's shared-message (SOHM) table, which could never
+    resolve because its index version byte was skipped.
+  - Two threads reading two chunked datasets through one `File` could get each
+    other's chunks (the shared chunk cache was switched between datasets
+    across separate lock acquisitions). The cache is now keyed by dataset.
+- `clawhdf5-format` reader — errors on valid files: enum and bool datasets
+  through the numeric readers; the "don't filter partial edge chunks" layout
+  flag; Fletcher32 ahead of deflate (NetCDF-4's order). Unknown-message flags
+  follow libhdf5 (`tbogus.h5`): "fail if unknown" is refused, "fail if unknown
+  and writing" is ignored by a reader.
+- `clawhdf5-format` writer — **files libhdf5 rejects or reads wrong:**
+  - Extensible Array (one unlimited dimension): chunks from index 244 on were
+    written but never indexed and read as 0, by libhdf5 and by us.
+  - Fixed Array: more than 1 024 chunks gave checksum errors (data blocks
+    were never paged).
+  - A finite max shape larger than the shape gave libhdf5 "addr overflow"; an
+    unlimited dimension that is not the first scrambled the data; several
+    unlimited dimensions (`(None, None)`) broke the whole file. These now
+    write the index libhdf5 writes (swizzled Extensible Array, or a B-tree v2
+    index for several unlimited dimensions).
+  - Header messages over 64 KiB (the size field is 16 bits) and compact
+    datasets at 65 534–65 535 bytes produced corrupt files.
+  - Reference, Opaque, BitField and Time datatypes were written as empty
+    messages; they now encode as HDF5 2.0 does.
+  - `with_page_size` wrote a nonexistent superblock version 4; it now writes
+    the v3 superblock and File Space Info message libhdf5 writes.
+  - `FillTime` values were rotated on disk (NEVER was written as ALLOC, and so
+    on). New `DatasetBuilder::with_fill_value`.
+  - An empty-string attribute got a zero-size datatype, which made every
+    attribute on the object unreadable in libhdf5.
+  - `maxshape` equal to the shape no longer forces chunked layout.
+- `clawhdf5-format`: **a truncated deflate chunk read back short, with no
+  error.** The deflate filter used flate2's streaming reader, which returns the
+  bytes it has when the input runs out before the end-of-stream marker. It now
+  decodes in one pass into a buffer sized to the chunk and reports a
+  truncated stream as `DecompressionError`. Same fix in `clawhdf5-filters`,
+  where output longer than the stated size was also silently cut off; it is
+  now an error.
+
+### Defaults
+- `clawhdf5-agent`: `MemoryConfig::float16` defaults to `true` for new stores,
+  measured rather than assumed: identical LongMemEval retrieval on real
+  embeddings, 48% smaller files and faster checkpoints and opens at 100K.
+  `clawhdf5-cli create --f32` opts out; like `--f32-index`, it only ever
+  switches the default off.
+- `clawhdf5-agent`: `MemoryConfig::quantized_index` defaults to `true` for new
+  stores. The reason it had been off — that int8 search was slower on ARM —
+  did not survive measurement (see Corrections). Stores that predate the
+  setting still load it as `false`, so reopening one never changes how its
+  index is held; a store written by the v2.5.0 CLI is now a test fixture that
+  guards exactly that, and the test fails if the load default is changed.
+- `clawhdf5-cli`: `create --f32-index` opts out. `create` used to assign
+  `--quantized-index` straight into the config, which under the new default
+  would have forced every CLI-created store back to f32 unless the caller
+  knew to ask; it now only ever switches the default off.
+
+### Performance
+- `clawhdf5-agent`: consolidation's novelty scoring (each `add_memory` against
+  the whole working tier) computes the new record's norm once, takes each
+  comparison in one vectorised pass instead of three, and splits a working
+  tier of 4 096+ records across threads — same results, tested against the
+  old formula. It had made `consolidation_efficiency` stall at 100K; the
+  complete run now takes 8 min and fills in the 100K cycle row (46.66 ms) and
+  the memory-reduction table.
+- `clawhdf5-bench`: `consolidation_efficiency` no longer prints a record-count
+  ratio as a "BM25 Speedup" (it was never measured), nor claims cycle time
+  grows sub-linearly (its own numbers grow slightly faster than linearly).
+- `clawhdf5-agent`: **knowledge-graph traversal was 6.5x slower than it
+  should be.** `bfs_neighbors` and `spreading_activation` built an adjacency
+  index over the whole graph on every call (1efd82c), so a 2-hop BFS over 1K
+  entities took 155 µs. The index is now cached on `KnowledgeCache` and
+  checked against a fingerprint of the graph on each use — one pass over
+  entity ids and relation endpoints, no allocation — so any change, including
+  direct edits of its public `Vec`s, still rebuilds it (tested). BFS over 1K
+  entities: 155.1 -> 23.1 µs; spreading activation over 100: 22.8 -> 10.1 µs.
+- `clawhdf5-format`, `clawhdf5-filters`: both deflate paths hand the codec the
+  whole chunk in one call, into a buffer allocated once, instead of streaming
+  it through a 32 KiB buffer: about 5% on chunked writes and 10% on zlib-ng's
+  1 MB inflate.
+- `clawhdf5-accel`: **`dot_i8` has aarch64 kernels** — `SDOT` for CPUs with
+  the ARMv8.2 dot-product extension (Cortex-A76 and later, Neoverse-N1, every
+  Apple Silicon generation) and plain NEON (`vmull_s8` + `vpadalq_s16`) for
+  the rest, selected at runtime. `SDOT` is issued through inline assembly,
+  because the `vdotq_s32` intrinsic is still behind the unstable
+  `stdarch_neon_dotprod` feature. On a Raspberry Pi 5 at N = 100 000 and
+  equal recall, the quantised index answers **1.18x the queries per second**
+  of f32 (7 267 vs 6 164) and builds **2.3x faster** (14 464 vs 33 413 ms).
+  Both kernels are tested bit-for-bit against scalar on real hardware, each
+  explicitly — dispatch only ever takes one path on a given CPU, so testing
+  through it alone would have left the plain-NEON fallback unexercised on any
+  machine with `SDOT`.
+
+### Corrections
+- The v2.7.0 entry for `dot_i8` said `quantized_index` stayed off by default
+  because "aarch64 falls back to the scalar loop", implying the ~13% search
+  penalty measured on x86 applied on ARM too. It did not. That figure came
+  from scalar int8 against hand-written AVX2 f32 kernels on x86, whose
+  portable baseline is SSE2; on aarch64 NEON is the baseline, and measured on
+  a Pi 5 the scalar int8 loop already matched f32 for search while building
+  1.76x faster. The claim was extrapolated rather than measured.
+
 ## v2.7.0 (2026-09-20)
 
 ### Upgrade Notes

CLAUDE.md

@@ -1,7 +1,7 @@
 # clawhdf5
 
 ## Purpose
-Pure-Rust HDF5 format implementation with HNSW vector search, WAL-backed persistence, agent memory storage, and GPU-accelerated I/O. Used by ZeroClaw as its persistent memory and knowledge graph backend.
+Pure-Rust HDF5 format implementation with HNSW vector search, WAL-backed persistence, agent memory storage, and GPU-accelerated vector search. A standalone library. Its one verified consumer is ClawBrainHub (`.brain` files); no agent framework integrates it (OpenClaw and ZeroClaw claims were withdrawn on 2026-09-25 — neither was ever true).
 
 ## Architecture
 
@@ -11,15 +11,15 @@ Cargo workspace with 16 crates under `crates/` (plus `libaec-sys`, an internal F
 |-------|------|
 | `clawhdf5-format` | HDF5 binary spec parser (superblock, B-tree, heap) — also holds shared type definitions and physical constants |
 | `clawhdf5-io` | Read/write implementation |
-| `clawhdf5-filters` | Compression filters (gzip, LZ4, Zstd, Blosc) |
+| `clawhdf5-filters` | Deflate backends (zlib-rs, zlib-ng, Apple Compression); the HDF5 filter pipeline and the other codecs (LZ4, Zstd, SZIP, N-Bit, scale-offset, pcodec) live in `clawhdf5-format`. No Blosc. |
 | `clawhdf5-derive` | Proc-macro derive for HDF5-serializable structs |
 | `clawhdf5` | Main facade crate |
 | `clawhdf5-netcdf4` | NetCDF-4 compatibility layer |
 | `clawhdf5-ann` | HNSW approximate nearest-neighbor vector index |
 | `clawhdf5-agent` | Agent memory, session history, knowledge graph storage |
-| `clawhdf5-gpu` | GPU-accelerated I/O via wgpu (hand-written WGSL compute shaders) |
+| `clawhdf5-gpu` | GPU vector distance computation via wgpu (hand-written WGSL compute shaders) — not dataset I/O |
 | `clawhdf5-accel` | CPU SIMD acceleration path |
-| `clawhdf5-migrate` | Schema migration engine |
+| `clawhdf5-migrate` | SQLite → HDF5 agent-memory migration |
 | `clawhdf5-android` | Android JNI bindings |
 | `clawhdf5-cli` | Command-line interface |
 | `clawhdf5-napi` | Node.js native addon bindings |
@@ -27,7 +27,12 @@ Cargo workspace with 16 crates under `crates/` (plus `libaec-sys`, an internal F
 | `clawhdf5-bench` | Benchmark suite |
 
 ## Key Features
-- Zero-dependency HDF5 read/write (no libhdf5 C library required)
+- Zero-C-dependency HDF5 read/write: no libhdf5, and deflate defaults to
+  pure-Rust zlib-rs (`fast-deflate` opts into zlib-ng, which needs cmake).
+  `ci-test.sh` fails if a C-building crate enters the core crates' default
+  tree. flate2 must keep `runtime_detection` with zlib-rs — without it zlib-rs
+  loses SIMD and inflates 3.5x slower. MSRV is 1.92 (`rust-version`, checked
+  in CI).
 - HNSW vector index for semantic similarity search over agent memories — the
   `clawhdf5-agent` `hnsw` feature is **on by default**, so `hybrid_search` uses
   the approximate `clawhdf5-ann` index for the vector stage (the index mirrors
@@ -39,15 +44,20 @@ Cargo workspace with 16 crates under `crates/` (plus `libaec-sys`, an internal F
   (plain closest-M capped recall on clustered data: 0.31 recall@10 at 100K). Its
   graph is saved to `<store>.h5.ann` at each checkpoint and reloaded by `open()`
   (tied to the checkpoint by a generation id; stale/damaged sidecars are
-  ignored and the index rebuilt). `MemoryConfig::quantized_index` (off by
-  default, persisted) stores the index's own copy of the embeddings as `i8`,
+  ignored and the index rebuilt). `MemoryConfig::quantized_index` (**on by
+  default** for new stores, persisted; stores predating the setting load as
+  `false` and keep their f32 index — guarded by
+  `tests/fixtures/store_v2_5_0.h5`; CLI opt-out is `create --f32-index`)
+  stores the index's own copy of the embeddings as `i8`,
   which roughly halves a loaded store's memory (2.72x -> 1.74x the raw vectors
   at 100K); because quantised distances are approximate and `ef` cannot
   compensate, the query path then re-scores the candidate pool against the
-  exact embeddings, which holds recall at the f32 index's level. On AVX2 it is
-  also 1.63x the QPS and 1.8x the build speed (`clawhdf5_accel::dot_i8`); it
-  stays off by default only because that kernel is AVX2-only and aarch64 falls
-  back to scalar. `hybrid_search` keeps one incremental BM25
+  exact embeddings, which holds recall at the f32 index's level. It is also
+  faster at equal recall: 1.63x the QPS on x86-64 (AVX2) and 1.18x on a
+  Raspberry Pi 5 (`clawhdf5_accel::dot_i8`, NEON `SDOT` via inline asm since
+  the intrinsic is unstable; plain NEON on pre-dotprod cores). The aarch64
+  code is `cfg`'d out on x86, so x86 CI never compiles or lints it — test it
+  on real ARM (`rpivision02`, 10.0.2.3, is a Pi 5). `hybrid_search` keeps one incremental BM25
   index for the life of the store and never writes the store: Hebbian
   activation boosts are persisted by the next checkpoint (or on drop), not per
   query. Measure any search-path change with
@@ -77,8 +87,51 @@ Cargo workspace with 16 crates under `crates/` (plus `libaec-sys`, an internal F
   `export` do). An unreadable WAL (torn header, bad magic) is quarantined to
   `<store>.h5.wal.corrupt-<ts>` rather than blocking `open()`; a WAL with an
   unknown *newer* version still fails and is left untouched.
-- `MemoryConfig::compression` uses deflate by default; enable the agent's
-  `zstd` feature to compress embeddings with Zstd instead (links libzstd).
+- `MemoryConfig::float16` (**on by default** for new stores, persisted;
+  existing stores keep their recorded `false` — guarded by the v2.5.0
+  fixture in `tests/float16_store.rs`; CLI opt-out is `create --f32`) writes
+  `/memory/embeddings` as IEEE half precision (48% smaller file at 100K;
+  LongMemEval with real MiniLM embeddings identical to f32).
+  `MemoryCache::half_precision` rounds each embedding as it enters the cache (push, update, WAL replay, and on load of a store still
+  `f32` on disk), so memory and file agree bit for bit; the conversions live
+  in `clawhdf5_format::float16` and must stay the single implementation.
+  Values beyond ±65504 are `MemoryError::InvalidEntry`. Interop: every file
+  must open in h5py — `f32` datasets and empty datasets did not until
+  2026-09-23 (see `docs/known-issues.md`); the agent's `h5py_interop` test
+  guards a whole store.
+- `HDF5Memory::search(query_emb, text, &SearchOptions)` is the full search
+  path: optional source-channel filter (applied before ranking; exact scan of
+  the allowed records whenever cheaper than `pool × M` index distance
+  evaluations, and as the fallback when the pool comes back short), fusion,
+  activation scaling, optional re-ranking and confidence rejection.
+  `hybrid_search`/`hybrid_search_with` are thin wrappers; `ClawhdfBackend`
+  (the `openclaw` module) is `search` with re-rank + confidence on.
+- **OpenClaw is not supported** (decided 2026-09-25): clawhdf5 is not an
+  OpenClaw memory plugin and never was — the old `memory.backend = "clawhdf5"`
+  config was never valid. Don't reintroduce OpenClaw claims; `docs/openclaw.md`
+  records what a real plugin would need.
+- **ZeroClaw does not use clawhdf5** (checked 2026-09-25 against upstream
+  v0.8.5 and the `osobh/zeroclaw` fork, and their full history): no
+  `clawhdf5` feature or backend exists; ZeroClaw's memory backends are
+  sqlite/lucid/postgres/qdrant/markdown/none behind its own `Memory` trait.
+  `clawhdf5-migrate`'s default SQLite layout (`memory_chunks`, `sessions`,
+  `entities`, `relations`) is not ZeroClaw's schema either (ZeroClaw's is a
+  `memories` table). Don't reintroduce integration claims without an
+  integration and a test against the real consumer. Measure changes with
+  `search_harness --options-study`.
+- `MemoryConfig::compression` is off by default; when on, embeddings are
+  deflate-compressed, or Zstd with the agent's `zstd` feature (links libzstd).
+- Signed checkpoints (`clawhdf5-agent` `signing` module): with
+  `HDF5Memory::set_signing_key` every checkpoint stores an Ed25519-signed
+  manifest (SHA-256 per record in a Merkle tree + settings/sessions/graph
+  hashes; per-record hashes in `/integrity/record_hashes`);
+  `HDF5Memory::verify(path, &pk)` locates edits. The hashes must cover exactly
+  what the file persists in the form the loader returns it (strings lose
+  trailing NULs; an empty WAL mark is not written) or untouched stores stop
+  verifying — `tests/signed_store.rs` round-trips awkward strings. The key is
+  never persisted; a signed store refuses to checkpoint without it
+  (`MemoryError::SigningKeyRequired`, and `MemoryError` is `#[non_exhaustive]`).
+  WAL entries after the checkpoint are not covered.
 - `Dataset::verify_provenance()` (clawhdf5 facade, `provenance` feature, on by
   default) recomputes a dataset's SHA-256 and compares it against the
   `_provenance_sha256` attribute written automatically on save when
@@ -95,7 +148,7 @@ Cargo workspace with 16 crates under `crates/` (plus `libaec-sys`, an internal F
   Alerts never block a save — drain them with `HDF5Memory::take_anomaly_alerts`.
   `MemorySource` for this bookkeeping is inferred from the caller-supplied
   `source_channel` string (a heuristic, not an authenticated trust boundary).
-- GPU-accelerated batch I/O for large dataset processing
+- GPU-accelerated vector distance computation (`clawhdf5-gpu`, wgpu); HDF5 I/O itself is CPU-only
 - Python and Node.js bindings for cross-language use
 - NetCDF-4 compatibility for scientific data interop
 
@@ -111,6 +164,24 @@ cargo build --release
 cargo test --workspace
 ```
 
+### CI
+`.gitea/workflows/ci.yml` has two jobs, both green as of 2026-09-22:
+- **`test`** (`ubuntu-latest`, in `rust:latest`) runs `scripts/ci-test.sh` with
+  the h5py/netCDF4 interop suites required (`CLAWHDF5_REQUIRE_INTEROP=1`).
+  Served by the `tank` and `architect` runners.
+- **`test-arm64`** (`linux_arm64`) lints and tests the aarch64 code — the NEON
+  kernels are `cfg`'d out on x86, so this is the only place they are built.
+  Served by `vision-01` (host mode) and `vision-02` (Docker), so steps must
+  work in both.
+
+Keep workflows free of JavaScript actions (`actions/checkout`, `actions/cache`,
+…): `rust:latest` has no `node`, and not every runner reaches GitHub, where
+they are fetched from. Check out with plain `git` instead. The `test` job
+installs `cmake` for the opt-in `fast-deflate` (zlib-ng) steps; the default
+build needs no C toolchain, so `test-arm64` does not.
+All runners are on `gitea-runner` 3.5.0, from `docker.gitea.com/act_runner`
+— `gitea/act_runner:latest` on Docker Hub is frozen at 0.6.1.
+
 ### CLI
 ```bash
 cargo run -p clawhdf5-cli -- --help
@@ -125,4 +196,12 @@ python -c "import clawhdf5; print(clawhdf5.__version__)"
 ```
 
 ## Integration
-ZeroClaw imports this as a Cargo feature (`clawhdf5` feature flag) to persist agent memory with HNSW vector search for context retrieval.
+- **ClawBrainHub** (`clawverse/clawbrainhub` on git.redclaw.dev) is the one
+  verified consumer: `cbh-core` reads and writes `.brain` files through the
+  facade (`File`, `FileBuilder`, `AttrValue`, `Selection`), `cbh-scanner`
+  uses the facade, and `cbh-cli` uses `clawhdf5_agent::bm25::BM25Index`. It
+  depends on this repo by path (`../clawhdf5`), so it builds against whatever
+  is checked out — changes to those APIs reach it directly. Verified
+  2026-09-25 against main: builds, and its 204 tests pass.
+- OpenClaw and ZeroClaw were both described as consumers; neither integrates
+  clawhdf5 (see Key Features and `docs/openclaw.md`).

Cargo.toml

@@ -23,6 +23,9 @@ resolver = "2"
 [workspace.package]
 version = "2.7.0"
 edition = "2024"
+# Oldest toolchain that builds the whole workspace; CI checks it. wgpu (in
+# clawhdf5-gpu) requires 1.92.
+rust-version = "1.92"
 license = "MIT"
 repository = "https://git.redclaw.dev/quantumclaw/clawhdf5"
 

README.md

@@ -3,24 +3,102 @@
 **The memory layer AI agents deserve. One file. Pure Rust. Zero C dependencies.**
 
 [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE)
-[![Rust](https://img.shields.io/badge/rust-1.75%2B-orange.svg)](https://www.rust-lang.org)
-[![Tests](https://img.shields.io/badge/tests-1650%2B%20passing-brightgreen.svg)](#performance)
-[![LongMemEval](https://img.shields.io/badge/LongMemEval%20oracle-Turn--Level%20Hit@5%2084%25%20BM25--only-blue.svg)](BENCHMARKS.md#longmemeval-results)
-[![Footprint](https://img.shields.io/badge/footprint-6.5%20KB%2Frecord-lightgrey.svg)](BENCHMARKS.md#memory-footprint)
+[![Rust](https://img.shields.io/badge/rust-1.92%2B-orange.svg)](https://www.rust-lang.org)
+[![Tests](https://img.shields.io/badge/tests-1850%2B-brightgreen.svg)](#building)
+[![LongMemEval](https://img.shields.io/badge/LongMemEval__s-Turn--Level%20Hit@5%2081.4%25%20hybrid-blue.svg)](BENCHMARKS.md#longmemeval-results)
+[![Footprint](https://img.shields.io/badge/on--disk-~820%20B%2Frecord%20float16%2C%20synthetic%20text-lightgrey.svg)](BENCHMARKS.md#memory-footprint-1)
 
-ClawHDF5 is a pure-Rust HDF5 implementation combined with a research-grade agent memory engine. It gives AI agents persistent, searchable, cryptographically verifiable memory — all stored in a single portable file.
+ClawHDF5 is a pure-Rust HDF5 implementation combined with a research-grade agent memory engine. It gives AI agents persistent, searchable, cryptographically verifiable memory (Ed25519-signed checkpoints) — all stored in a single portable file.
 
 > **Two things live here:**
 > - **A general-purpose, pure-Rust HDF5 library** — zero C dependencies, NetCDF-4 support, SIMD/GPU acceleration. See the **[Crate Map](#crate-map)** and **[BENCHMARKS.md](BENCHMARKS.md)** for the libhdf5 head-to-head numbers.
 > - **An agent memory layer built on top of it** — vector search, knowledge graph, hippocampal-style consolidation, in `clawhdf5-agent`.
 
-```
-cargo add clawhdf5                          # core HDF5 read/write, no agent layer
-cargo add clawhdf5-agent --features agent   # + agent memory layer
+The crates are not on crates.io yet, so depend on them from git:
+
+```toml
+[dependencies]
+clawhdf5       = { git = "https://git.redclaw.dev/quantumclaw/clawhdf5" }  # core HDF5 read/write
+clawhdf5-agent = { git = "https://git.redclaw.dev/quantumclaw/clawhdf5" }  # + agent memory layer
 ```
 
+> **C dependencies, precisely:** the core crates (`clawhdf5`, `clawhdf5-agent`,
+> `-format`, `-io`, `-filters`, `-ann`, `-accel`, `-netcdf4`, `-cli`) build no C
+> code by default — no libhdf5, and deflate is the pure-Rust
+> [zlib-rs](https://github.com/trifectatechfoundation/zlib-rs), which matches
+> zlib-ng on HDF5 reads and writes and produces byte-identical output
+> ([BENCHMARKS.md § Deflate backend](BENCHMARKS.md#deflate-backend-zlib-rs-vs-zlib-ng)).
+> CI fails if a C-building crate enters their default dependency tree. C comes
+> in only when you ask for it: `fast-deflate` (zlib-ng, needs cmake), `zstd`,
+> `szip`, the BLAS backends, `clawhdf5-migrate` (bundled SQLite) and the
+> Node.js bindings.
+
 > **New here?** Start with the **[Quickstart Guide](docs/QUICKSTART.md)** · See **[Use Cases](docs/USE_CASES.md)** · Read **[Benchmarks](BENCHMARKS.md)**
 
+## What's new (v2.2 → v2.7, and unreleased)
+
+Five releases in September 2026. Details, including upgrade notes and every
+breaking change, are in [CHANGELOG.md](CHANGELOG.md).
+
+**HDF5 correctness (read these if you read files with an earlier release)**
+- **Extensible Array chunk indexes returned wrong data** past the 36th chunk —
+  any dataset with one unlimited dimension. Silent: plausible numbers from the
+  wrong chunks. Fixed in v2.7.0; re-read affected data.
+- Fixed and Extensible Array checksums are now verified, so a corrupt chunk
+  index is `ChecksumMismatch` instead of wrong data (v2.7.0).
+- Compound datatypes written with default libver bounds (plain
+  `h5py.File(path, 'w')`) were mis-parsed; HDF5 2.0 compound v5 and native
+  complex (class 11) types now parse (v2.2.0–v2.3.0).
+- Committed datatypes, fill values, soft links and `H5T_STD_REF` references now
+  read correctly; external links and external raw data are explicit errors;
+  `attrs()` no longer silently drops attributes (v2.3.0–v2.5.0).
+- Datasets indexed by a version-2 B-tree now read (v2.5.0).
+
+**Security and robustness**
+- A crafted file could abort any reader via B-tree v2 recursion or explode it
+  via shared children; both are now fast errors (v2.7.0).
+- Virtual-dataset source paths are confined to the file's directory; chunked
+  reads use overflow-checked sizes and fallible allocation, and the facade
+  writes files atomically (v2.3.0).
+- Agent store: single-writer lock plus `open_read_only`; a crash between
+  checkpoint and WAL truncate no longer duplicates entries; unreadable WALs are
+  quarantined instead of blocking `open()` (v2.3.0).
+
+**Search quality and speed**
+- HNSW neighbour selection now uses the paper's diversity heuristic: recall@10
+  at 100K went from 0.31 to 0.98 (v2.4.0).
+- `hybrid_search` is 79–190× faster than v2.3.0 (p50 0.07 ms at 1K, 4.65 ms at
+  100K). It no longer rebuilds BM25 or rewrites the store per query, and the
+  HNSW graph is persisted (v2.4.0).
+- Default fusion weights are now the measured 0.4 / 0.6 (v2.5.0). Re-ranking had
+  been discarding the retrieval score, costing the Markdown backend 40.6pp of
+  Hit@1; fixed in v2.6.0.
+- Selection reads decode only the chunks they touch (a 64×64 window: 105 ms to
+  0.39 ms), and full reads are 1.2–1.9× faster (v2.5.0).
+
+**Memory**
+- A loaded store holds ~30% less (embeddings stored once, v2.6.0), and the
+  int8 HNSW index, **on by default for new stores** (unreleased), brings a
+  100K × 384 store to 1.74× the raw vectors. At equal recall it is also faster
+  than `f32`: 1.63× QPS on AVX2, 1.18× on a Raspberry Pi 5 (NEON `SDOT`).
+
+**Interop and search (unreleased)**
+- **Files we write now open in h5py and libhdf5.** Every `f32` dataset —
+  including every agent store's embeddings — and every empty dataset was
+  refused by libhdf5. Both were write-side bugs in every release; agent stores
+  fix themselves at their next checkpoint. See
+  [docs/known-issues.md](docs/known-issues.md).
+- `MemoryConfig::float16` now stores half-precision embeddings (it was
+  ignored), and is on by default for new stores: 48% smaller files, and
+  identical LongMemEval retrieval on real embeddings.
+- `HDF5Memory::search` with `SearchOptions`: filter by source channel (exact
+  filtered top-k, never slower than unfiltered), and opt-in re-ranking and
+  confidence rejection, which used to be reachable only through `ClawhdfBackend`.
+
+**Tooling**
+- CI now runs the h5py/netCDF4 interop suites for real (they had been skipping
+  silently) and runs an aarch64 job for the NEON kernels.
+
 ---
 
 ## Why ClawhDF5?
@@ -33,16 +111,16 @@ Every AI agent needs memory. Today that means scattered Markdown files, SQLite d
 | Keyword search | Separate FTS engine | Integrated BM25 |
 | Knowledge graph | Neo4j or none | In-file graph with spreading activation |
 | Memory consolidation | Manual pruning | Hippocampal-inspired automatic tiers |
-| Temporal queries | Custom code | Native temporal index (716ns) |
-| Multi-modal | Multiple stores | Unified cross-modal search |
-| Security | Hope for the best | Provenance tracking + anomaly detection |
+| Temporal queries | Custom code | Native temporal index (622 ns range query over 10K) |
+| Multi-modal | Multiple stores | Unified cross-modal search (exact scan: 842 µs over 1K records) |
+| Integrity | Hope for the best | Ed25519-signed checkpoints that pinpoint any edited record, chained-CRC WAL, checksummed chunk indexes, write-anomaly alerts |
 | Portability | Config + DB + files | **One `.h5` file. Copy it anywhere.** |
 
 ---
 
 ## Performance
 
-Vector search and agent-memory operations below are benchmarked on Intel i7-12650H (10C/16T), 384-dim embeddings, Criterion.rs. The HDF5 Core I/O table immediately below is from a separate, independently reproduced run (see its own hardware note).
+The brute-force/IVF vector search, agent-memory, on-disk footprint and consolidation figures below were measured 2026-09-24 on tank (AMD Ryzen 7 7800X3D, 8C/16T), commit 5c8323c, 384-dim embeddings; the commands are in [BENCHMARKS.md](BENCHMARKS.md). Exceptions are marked where they appear: the HDF5 Core I/O table immediately below is from a separate, independently reproduced run (see its own hardware note), and the HNSW `f32`/`i8` table and the in-memory `i8` column were not re-measured on 2026-09-24.
 
 ### HDF5 Core I/O (vs libhdf5 1.14.6)
 
@@ -58,31 +136,63 @@ Figures below are from an independent reproduction run on a second machine (AMD
 | Sequential read (100K f32) | 23.3 µs | 63.6 µs | **2.7×** |
 | Sequential write (100K f32) | 210 µs | 189 µs | **≈ tie** |
 
+The chunked-write row was re-measured on the same machine on 2026-09-23, after
+the default deflate backend became pure-Rust zlib-rs: 1.46 ms against
+libhdf5's 51.4 ms (**35×**), and 1.48 ms with zlib-ng. libhdf5's own time on
+that machine moved from 65.0 to 51.4 ms between the two dates, which is most
+of the difference from 45×; compare same-day numbers only.
+
 ### Vector Search
 
-| Scale | Flat | IVF (nprobe=10) | IVF-PQ | vs MemX¹ |
-|-------|------|-----------------|--------|----------|
-| 1K | **54 µs** | — | — | — |
-| 10K | 753 µs | **27 µs** | — | — |
-| 100K | 11.4 ms | 1.32 ms | **1.19 ms** | ~8–76× (see caveat) |
+**HNSW (the default backend for `hybrid_search`)** — `search_harness`, clustered
+384-dim data, M = 16, ef_construction = 64, recall measured against an exact scan.
+See [BENCHMARKS.md § Search harness](BENCHMARKS.md#search-harness-baseline-v230)
+and [§ Quantising the index copy](BENCHMARKS.md#quantising-the-index-copy-quantized_index):
 
-> Reproduced on the same second machine (Ryzen 7 7800X3D) with a corrected,
-> apples-to-apples SIMD/scalar/parallel comparison methodology — see
-> [BENCHMARKS.md § Independent Validation: tank — LongMemEval & Vector
-> Search](BENCHMARKS.md#independent-validation-tank--longmemeval--vector-search-ryzen-7-7800x3d-2026-08-05).
+| N = 100K, ef = 64 | recall@10 | QPS | build |
+|---|---:|---:|---:|
+| `f32` index | 0.9945 | 13 399 | 3.2 s |
+| `i8` index + exact re-score (**default for new stores**) | 0.9940 | **21 848** | **1.8 s** |
+
+Before the v2.4.0 neighbour-selection fix, recall@10 at 100K was 0.31. These
+two rows are a paired comparison (medians of alternating runs, same binary).
+A single `f32` run on 2026-09-24 measured recall 0.9945, 19 001 QPS and a
+2.7 s build; the int8 row was not re-run, so the pair has not been re-checked
+([§ Quantising the index copy](BENCHMARKS.md#quantising-the-index-copy-quantized_index)).
+
+**Brute-force and IVF paths** (Criterion, tank, 2026-09-24):
+
+| Scale | Flat | IVF (nprobe=10) | IVF-PQ | MemX¹ (claimed, end-to-end) |
+|-------|------|-----------------|--------|----------|
+| 1K | **47.4 µs** | — | — | — |
+| 10K | 500.5 µs | **24.8 µs** | — | — |
+| 100K | 6.58 ms | 592 µs | **869 µs** | <90 ms |
+
+> These replace figures from the original i7-12650H run (flat 54 µs / 753 µs /
+> 11.4 ms); a 2026-08-05 run on tank had already matched the new ones — see
+> [BENCHMARKS.md § Vector Search Latency](BENCHMARKS.md#vector-search-latency).
 
 ### Agent Memory Operations
 
 | Operation | Latency | Scale |
 |-----------|---------|-------|
-| Hybrid search (RRF) | **222 µs** | 1K records |
-| BM25 keyword search | **67 µs** | 1K records |
-| Knowledge graph BFS | **24 µs** | 1K entities |
-| Spreading activation | **17 µs** | 100 entities |
-| Temporal range query | **716 ns** | 10K timestamps |
-| Consolidation cycle | **164 µs** | 1K records |
-| Memory write (WAL) | **18 µs** | per record (group-commit append; HDF5 batched at flush) |
-| Importance gate | **61 ns** | per record |
+| Hybrid search (`HDF5Memory::hybrid_search`, p50) | **0.07 ms** / 0.49 ms / 4.69 ms | 1K / 10K / 100K records |
+| BM25 keyword search | **20.4 µs** | 1K records |
+| Knowledge graph BFS | **23.1 µs** | 1K entities |
+| Spreading activation | **10.1 µs** | 100 entities |
+| Temporal range query | **622 ns** | 10K timestamps |
+| Consolidation cycle | **115.2 µs** | 1K records |
+| Cross-modal search (exact scan, 2 embeddings per record) | **842.0 µs** / 8.44 ms | 1K / 10K records |
+| Memory write (WAL) | **26.1 µs** | per record (group-commit append; HDF5 batched at flush) |
+| Importance gate | **57.6 ns** | per record (trivial skip) |
+
+The old 18 µs WAL write was undated, from another machine: v2.3.0 measures
+24.3 µs on the same hardware as this table, the same as an `f32` store today.
+`float16` stores (the new default) add ~2 µs for rounding; the int8 index adds
+nothing. See [BENCHMARKS.md § Write Path](BENCHMARKS.md#write-path).
+Knowledge-graph traversal was briefly 6.5x slower (155 µs) until this re-run
+found and fixed an adjacency index rebuilt on every traversal; see
+[§ Knowledge Graph](BENCHMARKS.md#knowledge-graph).
 
 ### Chunked Write Throughput (codec comparison)
 
@@ -97,7 +207,7 @@ by default (AoS→SoA byte transpose, +157–204% throughput for float data):
 
 Use `.with_zstd(3)` or `.with_deflate(6)` for write-heavy workloads — both now perform at ~720–750 MiB/s on large matrices. Use `.with_pcodec()` for write-once/read-many workloads where compression ratio matters more than encode speed. Disable auto-shuffle with `.without_shuffle()` for byte arrays that don't benefit from AoS→SoA transposition.
 
-> ¹ MemX ([arxiv:2603.16171](https://arxiv.org/abs/2603.16171), March 2026): Rust + libSQL, claims <90ms at 100K records. **Not like-for-like:** MemX's figure is *end-to-end* (embeddings + FTS5 + four-factor re-ranking); ours is a *single component* (raw vector search). The ratio overstates the real advantage by an unquantified margin — order-of-magnitude indication only. See [BENCHMARKS.md](BENCHMARKS.md#comparison-to-memx-arxiv260316171).
+> ¹ MemX ([arxiv:2603.16171](https://arxiv.org/abs/2603.16171), March 2026): Rust + libSQL, claims <90ms at 100K records. **Not like-for-like:** MemX's figure is *end-to-end* (embeddings + FTS5 + four-factor re-ranking); ours is a *single component* (raw vector search), so the two columns are not comparable and no ratio is given. See [BENCHMARKS.md](BENCHMARKS.md#comparison-to-memx-arxiv260316171).
 
 ### LongMemEval Retrieval Recall
 
@@ -115,13 +225,17 @@ declaration:
 
 Hybrid is the strongest configuration, which is what running two retrieval stages
 is for. The weights matter more than the stages: a sweep of `vector_weight` from
-0.0 to 1.0 found the long-standing `0.7/0.3` default is **strictly dominated** by
-`0.4/0.6` — better on Hit@1, Hit@5, Hit@10 and MRR at both granularities. Use
-`0.4/0.6`, or `0.3/0.7` if rank-1 precision matters most. See
-[BENCHMARKS.md § Weight sweep](BENCHMARKS.md#longmemeval-results).
+0.0 to 1.0 found the old `0.7/0.3` default is **strictly dominated** by
+`0.4/0.6` — better on Hit@1, Hit@5, Hit@10 and MRR at both granularities. Since
+v2.5.0 `0.4/0.6` is the default (`hybrid::DEFAULT_FUSION`, used by
+`unified_search`, `hybrid_search_with` and `ClawhdfBackend`); callers that
+pass weights to `hybrid_search` explicitly choose their own. Use `0.3/0.7` if
+rank-1 precision matters most. Reciprocal rank fusion is selectable
+(`hybrid::Fusion::Rrf`) but measured worse than the weighted sum. See
+[BENCHMARKS.md § Weight sweep](BENCHMARKS.md#weight-sweep--full-haystack-n500).
 
-Vector embeddings require `--features embeddings`; without it the vector stage is
-inert and only the BM25 row is produced, which is what every previously published
+The benchmark's vector stage requires `clawhdf5-bench`'s `embeddings` feature
+(real MiniLM embeddings); without it the vector stage is inert and only the BM25 row is produced, which is what every previously published
 number here measured.
 
 On the easier `longmemeval_oracle` variant (evidence sessions only) the same
@@ -146,19 +260,53 @@ retrieval recall reported as QA accuracy typically overstates by 20–30 points.
 
 ### Memory Footprint
 
-| Records | File Size | Bytes/Record | With Compression |
-|---------|-----------|--------------|------------------|
-| 1K | ~6.5 MB | ~6.5 KB | ~2.1 MB (3.1x) |
-| 10K | ~65 MB | ~6.5 KB | ~21 MB (3.1x) |
-| 100K | ~645 MB | ~6.5 KB | ~208 MB (3.1x) |
+**On disk** — 384-dim `float16` embeddings (the default for new stores),
+200-char text, `footprint_bench`
+([BENCHMARKS.md § Memory Footprint](BENCHMARKS.md#memory-footprint-1)):
+
+| Records | File Size | Bytes/Record | Gzip-6 compressed |
+|---------|-----------|--------------|-------------------|
+| 1K | 810.4 KB | 829 B | 56.4 KB |
+| 10K | 7.8 MB | 820 B | 471.3 KB |
+| 100K | 76.7 MB | 803 B | 4.5 MB |
+
+The benchmark's synthetic embeddings and text are far more repetitive than
+real data (only 40 distinct texts), so no column here is an expectation for
+real data. The compressed column is an upper bound, and the Bytes/Record
+column is optimistic too: it is not an uncompressed figure, because the store
+always deflates its text (any string dataset of 4 KiB or more) whatever
+`MemoryConfig::compression` says. The `float16` embeddings alone are 768 B per
+record, so 200 characters of real text would take a record above 820 B.
+This table used to show `f32` stores (1.7 KB per record, 169.8 MB at 100K);
+those were not re-measured. The float16 study compares the two on the same
+data: 100K × 384 records take 80.8 MiB as `float16` and 154.0 MiB as `f32`.
+
+**In memory** — a store reopened from disk, 384-dim `f32`, measured with a
+counting allocator ([BENCHMARKS.md § Memory footprint](BENCHMARKS.md#memory-footprint)):
+
+| Records | Raw vectors | Reopened, `f32` index | Reopened, `i8` index (default) |
+|---------|-------------|-----------------------|--------------------------------|
+| 1K | 1 MiB | 4 MiB (2.40x) | 2 MiB (1.64x) |
+| 10K | 15 MiB | 44 MiB (3.03x) | 27 MiB (1.81x) |
+| 100K | 146 MiB | 399 MiB (2.72x) | **256 MiB (1.74x)** |
+
+Down from 505 MiB (3.44x) at 100K before v2.6.0, when the cache held every
+embedding twice. The `f32` column was re-measured on 2026-09-24 and reproduced
+exactly; the `i8` column was not re-run.
 
 ### Consolidation Efficiency
 
+1,000 records (10 signal + 990 noise), `working_capacity = 100`
+([BENCHMARKS.md § Consolidation Efficiency](BENCHMARKS.md#consolidation-efficiency)):
+
 | Metric | Before | After | Delta |
 |--------|--------|-------|-------|
-| Records in store | 1,000 | ~110 | −89% |
-| Hit@1 recall | ~60% | ~90% | +30% |
-| Search latency | ~2.8 ms | ~0.3 ms | **9x faster** |
+| Records in store | 1,000 | 100 | −90% |
+| Hit@1 recall (signal records) | 100% | 100% | no loss |
+| Search latency (avg) | 2.22 ms | 0.24 ms | **9.3x faster** |
+
+The consolidation cycle that does this took 0.13 ms; a cycle over 10K records
+takes 2.81 ms and over 100K 46.7 ms.
 
 **Full benchmark details: [BENCHMARKS.md](BENCHMARKS.md)**
 
@@ -166,74 +314,74 @@ retrieval recall reported as QA accuracy typically overstates by 20–30 points.
 
 ## Agent Memory Architecture
 
-ClawhDF5's agent memory engine implements research from 15+ recent papers on agentic memory systems. It's not a toy — it's the real thing.
+ClawhDF5's agent memory engine draws on 15+ recent papers on agentic memory systems (see [Research Foundation](#research-foundation)).
 
 ```
                          ┌─────────────────┐
                          │   Agent Query   │
                          └────────┬────────┘
                                   │
-                    ┌────────────▼────────────┐
-                    │   Hybrid Retrieval      │
-                    │  Vector + BM25 + RRF    │
-                    └────────────┬────────────┘
-                                 │
-              ┌──────────────────▼──────────────────┐
-              │         Multi-Factor Re-Ranking      │
-              │  temporal · authority · activation    │
-              └──────────────────┬──────────────────┘
-                                 │
-                    ┌────────────▼────────────┐
-                    │  Confidence Rejection   │
+                ┌─────────────────▼──────────────────┐
+                │ HDF5Memory::search                 │
+                │ optional source-channel filter     │
+                │ HNSW vector + BM25 keyword         │
+                │ weighted fusion (0.4 / 0.6)        │
+                │ × √(Hebbian activation)            │
+                └─────────────────┬──────────────────┘
+                                  │  opt-in (SearchOptions);
+                                  │  ClawhdfBackend turns both on
+                ┌─────────────────▼──────────────────┐
+                │ Multi-factor re-ranking            │
+                │ relevance · recency · authority ·  │
+                │ activation                         │
+                ├────────────────────────────────────┤
+                │ Confidence rejection               │
                 │ (suppress bad matches)             │
-                    └────────────┬────────────┘
+                └─────────────────┬──────────────────┘
                                   │
-        ┌────────────────────────▼────────────────────────┐
-        │              Memory Store (HDF5)                │
-        │                                                 │
-        │  ┌───────────┐ ┌───────────┐ ┌───────────────┐  │
-        │  │ Working   │→│ Episodic  │→│  Semantic     │  │
-        │  │ (bounded) │ │ (bounded) │ │ (long-term)   │  │
-        │  └───────────┘ └───────────┘ └───────────────┘  │
-        │                                                 │
-        │  ┌──────────┐ ┌──────────┐ ┌────────────────┐   │
-        │  │Knowledge │ │Temporal  │ │  Multi-Modal   │   │
-        │  │  Graph   │ │  Index   │ │  Embeddings    │   │
-        │  └──────────┘ └──────────┘ └────────────────┘   │
-        │                                                 │
-        │  ┌──────────┐ ┌──────────┐ ┌────────────────┐   │
-        │  │Provenance│ │ Anomaly  │ │   Source       │   │
-        │  │ Tracking │ │Detection │ │  Isolation     │   │
-        │  └──────────┘ └──────────┘ └────────────────┘   │
-        └─────────────────────────────────────────────────┘
-                              │
-                     ┌────────┴────────┐
-                     │ agent_memory.h5 │
-                     │   single file   │
-                     └─────────────────┘
+     ┌────────────────────────────▼────────────────────────────┐
+     │ In memory                                               │
+     │  cache (flat f32 embeddings) · BM25 index · HNSW index  │
+     │  provenance ledger + anomaly alerts (session-scoped)    │
+     └────────────────────────────┬────────────────────────────┘
+                                  │ WAL append; checkpoint
+     ┌────────────────────────────▼────────────────────────────┐
+     │ agent_memory.h5   /meta · /memory · /sessions ·         │
+     │                   /knowledge_graph                      │
+     │ agent_memory.h5.wal   chained-CRC write-ahead log       │
+     │ agent_memory.h5.ann   HNSW graph (derived, rebuildable) │
+     │ agent_memory.h5.lock  single-writer lock                │
+     └─────────────────────────────────────────────────────────┘
 ```
 
+Consolidation tiers (Working → Episodic → Semantic), the knowledge-graph
+algorithms, temporal and multi-modal indexes are library components you drive
+directly; the store persists the records, sessions and graph they work over.
+
 ### Module Overview
 
 | Module | What It Does |
 |--------|-------------|
-| **`knowledge`** | Entity/relation graph with BFS traversal, spreading activation, fuzzy entity resolution |
-| **`consolidation`** | Three-tier memory (Working → Episodic → Semantic) with importance scoring and time-decay |
-| **`hybrid`** | Vector + BM25 fusion with Reciprocal Rank Fusion (RRF, k=60). The vector stage uses the HNSW index by default (`hnsw` feature, on by default); disable with `--no-default-features --features float16` for an exact linear scan |
-| **`reranker`** | Multi-factor re-ranking: temporal recency, source authority, activation weight |
-| **`confidence`** | Low-confidence rejection — suppresses spurious recalls when nothing matches |
+| **`knowledge`** | Entity/relation graph with BFS traversal, spreading activation, fuzzy (Levenshtein) entity resolution |
+| **`consolidation`** | Three-tier memory (Working → Episodic → Semantic) with importance scoring, novelty, and time-decay |
+| **`hybrid`** | Vector + BM25 fusion. Default is a min-max-normalised weighted sum, vector 0.4 / keyword 0.6 (`hybrid::DEFAULT_FUSION`, tuned on LongMemEval); RRF is available via `Fusion::Rrf` / `hybrid_search_with`. The vector stage uses the HNSW index by default (`hnsw` feature); disable with `--no-default-features --features float16` for an exact linear scan |
+| **`reranker`** | Multi-factor re-ranking: retrieval relevance (leads, weight 1.0), temporal recency, source authority, activation weight. Opt-in via `SearchOptions::with_rerank`; on in `ClawhdfBackend` |
+| **`confidence`** | Low-confidence rejection — suppresses spurious recalls when nothing matches. Opt-in via `SearchOptions::with_confidence`; on in `ClawhdfBackend` |
 | **`temporal`** | Sorted timestamp index, session DAG, entity timeline, temporal query hints |
 | **`multimodal`** | Cross-modal search across text/image/audio/video embeddings |
-| **`provenance`** | Source attribution, FNV-1a content hashing, integrity verification |
-| **`anomaly`** | Write rate limiting, 15 injection pattern detectors, source distribution analysis |
-| **`openclaw`** | OpenClaw integration: MemoryBackend trait, Markdown ↔ HDF5 conversion |
+| **`signing`** | Ed25519-signed checkpoints: SHA-256 per record in a Merkle tree, plus hashes of settings, sessions and the knowledge graph; `HDF5Memory::verify` names any edited record |
+| **`provenance`** | Source attribution and an unkeyed FNV-1a content hash per record, held in memory for the session, for detecting accidental corruption (not tamper-proof) |
+| **`anomaly`** | Write rate limiting, 15 injection-pattern detectors, source-distribution analysis. Alerts never block a save; drain them with `take_anomaly_alerts` |
+| **`openclaw`** | `ClawhdfBackend`: a Markdown-oriented backend (ingest by section, search, read back by path, export). Named for OpenClaw, but **not an OpenClaw plugin** — see [docs/openclaw.md](docs/openclaw.md) |
 | **`vector_search`** | Flat cosine, pre-normed, SIMD, BLAS, GPU, parallel search paths |
-| **`ivf` / `pq`** | IVF-PQ approximate nearest neighbor for billion-scale search |
-| **`bm25`** | BM25 keyword index with TF-IDF scoring |
+| **`ivf` / `pq`** | Standalone IVF and IVF-PQ indexes (benchmarked to 100K vectors); not used by `HDF5Memory`, whose ANN index is HNSW |
+| **`bm25`** | Incremental Okapi BM25 inverted index, kept for the life of the store; optional stemming |
+| **`query_expand`** | Synonym / acronym / temporal query expansion |
 | **`entity_extract`** | Rule-based entity extraction from text chunks into the knowledge graph |
-| **`wal`** | Write-ahead log for crash-safe persistence; each entry is CRC32-checked on replay, so a corrupted entry stops replay there instead of loading bad data |
+| **`wal`** | Write-ahead log (v4) with a chained CRC32 per entry, so a corrupted, reordered, duplicated or spliced entry stops replay; checkpoints record a WAL mark so nothing is applied twice. Appends are not fsynced |
 | **`memory_strategy`** | Pluggable strategies: save-every, semantic-shift, user-correction detection |
 | **`decision_gate`** | Sub-microsecond trivial/substantive classification |
+| **`ephemeral`** | In-memory TTL/LFU working tier |
 | **`async_memory`** | Tokio-based async wrapper over the memory store (`async` feature) |
 
 ---
@@ -265,7 +413,7 @@ assert_eq!(values, vec![22.5, 23.1, 21.8]);
 use clawhdf5_agent::{HDF5Memory, MemoryConfig, MemoryEntry, AgentMemory};
 
 // Create memory store
-let config = MemoryConfig::new("agent.h5", "my-agent", 384);
+let config = MemoryConfig::new("agent.h5".into(), "my-agent", 384);
 let mut memory = HDF5Memory::create(config)?;
 
 // Save a memory
@@ -278,13 +426,67 @@ memory.save(MemoryEntry {
     tags: "preference".into(),
 })?;
 
-// Search
-let results = memory.search(&query_embedding, 5)?;
+// Hybrid search: vector + BM25, weighted 0.4 / 0.6 (the measured default)
+let results = memory.hybrid_search(&query_embedding, "user preferences", 0.4, 0.6, 5);
 for result in results {
     println!("[{:.3}] {}", result.score, result.chunk);
 }
 ```
 
+### Search Options
+
+```rust
+use clawhdf5_agent::SearchOptions;
+use clawhdf5_agent::confidence::ConfidenceConfig;
+use clawhdf5_agent::reranker::ReRankConfig;
+
+// Only memories from these source channels; still a full page of k results.
+let work = memory.search(
+    &query_embedding,
+    "deadline",
+    &SearchOptions::new(5).with_sources(["slack", "email"]),
+);
+
+// Re-rank by relevance, recency, source authority and activation, then drop
+// low-confidence results — the pipeline ClawhdfBackend runs.
+let careful = memory.search(
+    &query_embedding,
+    "user preferences",
+    &SearchOptions::new(5)
+        .with_rerank(ReRankConfig::default())
+        .with_confidence(ConfidenceConfig::default()),
+);
+```
+
+### Signed Checkpoints
+
+```rust
+use clawhdf5_agent::signing;
+
+// Once, somewhere safe: keep the secret key, publish the public key.
+let key = signing::generate_key();
+let public = key.verifying_key();
+
+// Every checkpoint is signed from now on. The key is never written to disk;
+// a signed store refuses to checkpoint without it.
+memory.set_signing_key(key);
+memory.flush_wal()?;
+
+// Anyone holding the public key can check the file, e.g. after copying it.
+let report = HDF5Memory::verify(std::path::Path::new("agent.h5"), &public)?;
+assert!(report.is_valid());
+// On a tampered file: report.changed_records lists the records that differ.
+```
+
+The signature covers every record (text, embedding as stored, channel,
+timestamp, session, tags, deleted flag, activation), the store's settings,
+its sessions and its knowledge graph — a change made with any tool is caught.
+It covers checkpoints, not saves still in the WAL
+(`report.wal_entries_unsigned` counts those). CLI: `clawhdf5-cli keygen`,
+`--signing-key <file>` on writing commands, and `verify --public-key`.
+Signing adds about 20% to a checkpoint and 32 bytes per record to the file
+([BENCHMARKS.md § Signed checkpoints](BENCHMARKS.md#signed-checkpoints)).
+
 ### Knowledge Graph
 
 ```rust
@@ -309,8 +511,8 @@ let neighbors = kg.bfs_neighbors(alice, 2);  // 2-hop neighborhood
 let activated = kg.spreading_activation(&[alice], 0.5, 0.01, 5);
 
 // Entity resolution — fuzzy matching
-let resolved = kg.resolve_or_create("alice", "person", -1, 2);
-// Returns existing Alice entity (Levenshtein distance ≤ 2)
+let (id, created) = kg.resolve_or_create("alice", "person", -1, 2);
+// id == alice, created == false: matched the existing entity (Levenshtein distance ≤ 2)
 ```
 
 ### Memory Consolidation
@@ -321,15 +523,19 @@ use clawhdf5_agent::consolidation::*;
 let config = ConsolidationConfig::default();
 let mut engine = ConsolidationEngine::new(config);
 
-// Add memories — automatically scored for importance
-engine.add_memory("User prefers dark mode", vec![0.1, 0.2, ...], MemorySource::User);
-engine.add_memory("ok", vec![0.0, 0.0, ...], MemorySource::System);
+let now = 1_700_000_000.0; // seconds since the epoch
+
+// Add memories — automatically scored for importance.
+// Elevated sources (System, …) go through a separate, explicit API.
+let id = engine.add_memory("User prefers dark mode".into(), vec![0.1, 0.2, ...], UntrustedSource::User, now);
+engine.add_trusted_memory("ok".into(), vec![0.0, 0.0, ...], TrustedSource::System, now);
 
 // Access a memory (reactivates it)
-engine.access_memory(0);
+engine.access_memory(id, now);
 
 // Run consolidation cycle
-let stats = engine.consolidate();
+engine.consolidate(now);
+let stats = engine.get_stats();
 // Working memories promote to Episodic (if important enough)
 // Episodic memories promote to Semantic (if accessed enough)
 // Low-decay memories get evicted when tiers are full
@@ -351,19 +557,25 @@ let ids = index.range_query(1700000000.0, 1700010800.0);
 let recent = index.latest(10);
 ```
 
-### OpenClaw Integration
+### Markdown Backend
+
+`ClawhdfBackend` ingests Markdown by section and searches it with the full
+pipeline. It is a library API — clawhdf5 is **not** an OpenClaw memory plugin
+([docs/openclaw.md](docs/openclaw.md)). Sections stored this way carry no
+embedding, so their search is keyword-only unless you save records with
+vectors through `save_entry`.
 
 ```rust
 use clawhdf5_agent::openclaw::*;
 
 // Create backend
-let mut backend = ClawhdfBackend::create("memory.h5", "agent-1", 384)?;
+let mut backend = ClawhdfBackend::create(std::path::Path::new("memory.h5"), 384)?;
 
 // Ingest existing Markdown memory files
 let md = std::fs::read_to_string("MEMORY.md")?;
 let count = backend.ingest_markdown("MEMORY.md", &md)?;
 
-// Search (uses full pipeline: RRF → re-rank → confidence filter)
+// Search (full pipeline: weighted vector + BM25 fusion → re-rank → confidence filter)
 let results = backend.search("user preferences", &query_embedding, 5);
 
 // Export back to Markdown
@@ -375,22 +587,23 @@ let exported = backend.export_markdown("MEMORY.md")?;
 ## Crate Map
 
 ```
-clawhdf5 workspace (16 crates, ~92K lines of Rust; plus libaec-sys, an
-                     internal FFI bindings crate for the optional szip feature)
+clawhdf5 workspace (16 crates, ~86K lines of Rust in src/, ~104K with tests
+                     and benches; plus libaec-sys, an internal FFI bindings
+                     crate for the optional szip feature)
 │
 ├── Core HDF5
-│   ├── clawhdf5-format      — Binary parser/writer (no_std), shared type definitions
-│   ├── clawhdf5-io          — I/O abstraction (buffered, mmap, async)
+│   ├── clawhdf5-format      — Binary parser/writer (no_std-capable), shared type definitions
+│   ├── clawhdf5-io          — I/O abstraction (file/memory readers; optional mmap, async, HSDS, MPI)
 │   ├── clawhdf5-filters     — Fast deflate path (zlib-ng); lz4/zstd/pcodec/szip filters live in clawhdf5-format
 │   ├── clawhdf5-derive      — Proc macros
 │   ├── clawhdf5             — High-level API
 │   ├── clawhdf5-netcdf4     — NetCDF-4 support
-│   ├── clawhdf5-accel       — SIMD (NEON, AVX2, AVX-512)
+│   ├── clawhdf5-accel       — SIMD (AVX2, NEON incl. SDOT int8; AVX-512 behind `avx512`)
 │   └── clawhdf5-gpu         — GPU compute (wgpu, hand-written WGSL compute shaders)
 │
 ├── Agent Memory
-│   ├── clawhdf5-agent       — Memory engine (20.9K lines, 32 modules; WAL is CRC32-checked per entry)
-│   ├── clawhdf5-ann         — HNSW approximate nearest neighbor (default backend; optional `parallel` feature)
+│   ├── clawhdf5-agent       — Memory engine (24.7K lines, 32 modules; chained-CRC WAL)
+│   ├── clawhdf5-ann         — HNSW approximate nearest neighbor (default backend; f32 or int8 storage; `parallel` build)
 │   ├── clawhdf5-migrate     — SQLite → HDF5 migration
 │   ├── clawhdf5-android     — Android JNI bridge
 │   └── clawhdf5-cli         — CLI tool
@@ -411,10 +624,10 @@ ClawhDF5's agent memory design draws from 15+ recent papers:
 
 | Paper | Key Insight | ClawhDF5 Module |
 |-------|-------------|-----------------|
-| **MemX** (2026) | RRF + multi-factor re-ranking | `hybrid`, `reranker` |
-| **Graph-Native Cognitive Memory** (2026) | Graph-structured belief revision | `knowledge` |
+| **MemX** (2026) | Hybrid fusion + multi-factor re-ranking | `hybrid`, `reranker` |
+| **Graph-Native Cognitive Memory** (2026) | Graph-structured memory (weighted, timestamped relations; entity timelines) | `knowledge`, `temporal` |
 | **CraniMem** (2026) | Bounded hippocampal memory | `consolidation` |
-| **D-MEM** (2026) | Reward prediction error gating | `consolidation` |
+| **D-MEM** (2026) | Surprise-gated storage (implemented as a novelty score) | `consolidation` |
 | **SYNAPSE** (2025) | Spreading activation for recall | `knowledge` |
 | **RAGdb** (2025) | Zero-dependency edge RAG | Architecture |
 | **MemoryGraft** (2025) | Memory poisoning attacks | `anomaly`, `provenance` |
@@ -429,30 +642,45 @@ ClawhDF5's agent memory design draws from 15+ recent papers:
 
 | Flag | Default | Description |
 |------|---------|-------------|
-| `agent` | no | Full agent memory layer |
-| `float16` | **yes** | Half-precision embedding storage (2× compression) |
+| `float16` | **yes** | Half-precision cosine kernel (`cosine_similarity_f16`). Half-precision *storage* is the `MemoryConfig::float16` setting below, and needs no feature |
 | `hnsw` | **yes** | HNSW approximate vector index for `hybrid_search` (via `clawhdf5-ann`); disable for an exact linear scan |
-
-`MemoryConfig::hnsw_m`, `hnsw_ef_construction` and `hnsw_ef_search` tune the
-vector index (16 / 64 / scale-with-`k` by default) and are stored with the
-file.
-
-`MemoryConfig::quantized_index` (off by default) stores the HNSW index's own
-copy of the embeddings as `i8`, roughly halving a loaded store's memory
-(2.72x -> 1.74x the raw vectors at 100k x 384). Quantised distances are
-approximate, so the query path re-scores the candidate pool against the exact
-embeddings the store already holds, which keeps recall at the `f32` index's
-level. On AVX2 it is also **faster** — 1.63x the queries per second and 1.8x
-the build speed at equal recall — because the int8 kernel is SIMD too. It
-stays off by default only because that kernel is AVX2-only and aarch64 falls
-back to a scalar loop. See `BENCHMARKS.md`, "Quantising the index copy".
-| `parallel` | no | Rayon parallel search |
+| `parallel` | **yes** | Parallel HNSW bulk build (same graph, ~3× faster on 16 cores) and Rayon brute-force search strategies |
+| `zstd` | no | Compress embeddings with Zstd instead of deflate when `MemoryConfig::compression` is on (links libzstd) |
 | `fast-math` | no | BLAS matrix-vector multiply |
 | `accelerate` | no | Apple Accelerate / AMX (macOS) |
 | `openblas` | no | OpenBLAS (Linux) |
 | `gpu` | no | GPU search via wgpu |
 | `async` | no | Tokio async with background flush |
 
+To opt out of the parallel build: `--no-default-features --features float16,hnsw`.
+For an exact linear cosine scan instead of HNSW: `--no-default-features --features float16`.
+
+`MemoryConfig::hnsw_m`, `hnsw_ef_construction` and `hnsw_ef_search` tune the
+vector index (16 / 64 / scale-with-`k` by default) and are stored with the
+file.
+
+`MemoryConfig::quantized_index` (**on by default** for new stores) holds the
+HNSW index's own copy of the embeddings as `i8`, roughly halving a loaded
+store's memory (2.72x -> 1.74x the raw vectors at 100k x 384). Quantised
+distances are approximate, so the query path re-scores the candidate pool
+against the exact embeddings the store already holds, which keeps recall at the
+`f32` index's level. It is also **faster**: 1.63x the queries per second at
+equal recall on x86-64 (AVX2) and 1.18x on a Raspberry Pi 5 (NEON `SDOT`), with
+index builds 1.8x and 2.3x faster respectively. Stores created before the
+setting existed keep their `f32` index; opt out for new stores with
+`quantized_index = false` or `clawhdf5-cli create --f32-index`. See
+[BENCHMARKS.md § Quantising the index copy](BENCHMARKS.md#quantising-the-index-copy-quantized_index).
+
+`MemoryConfig::float16` (**on by default** for new stores) stores the
+embeddings on disk as IEEE half precision (numpy `float16`): at 100K × 384 the
+file drops from 154 to 81 MiB, checkpoints and opens get faster, and on the
+full LongMemEval haystack with real MiniLM embeddings every retrieval metric
+matches `f32`. Embeddings are rounded as they are saved, so the store searches
+the same before and after a reopen; values must lie within ±65504. Existing
+stores keep their setting. Opt out with `float16 = false` or
+`clawhdf5-cli create --f32` — e.g. for unnormalised vectors. See
+[BENCHMARKS.md § float16 embedding storage](BENCHMARKS.md#float16-embedding-storage-memoryconfigfloat16).
+
 ### `clawhdf5-format`
 
 | Flag | Default | Description |
@@ -461,26 +689,31 @@ back to a scalar loop. See `BENCHMARKS.md`, "Quantising the index copy".
 | `deflate` | yes | Deflate compression |
 | `checksum` | yes | Jenkins lookup3 verification |
 | `provenance` | yes | SHA-256 provenance attributes |
-| `fast-deflate` | **yes** | zlib-ng backend for faster deflate |
-| `system-zlib-decompress` | **yes** | Use the system zlib for decompression where available |
+| `zlib-rs` | **yes** | Pure-Rust deflate backend ([zlib-rs](https://github.com/trifectatechfoundation/zlib-rs)) |
+| `fast-deflate` | no | zlib-ng deflate backend instead (C; needs `cmake`). Overrides `zlib-rs` when both are on |
+| `system-zlib-decompress` | **yes** | Use Apple's system libz for decompression (macOS only; no effect elsewhere) |
 | `parallel` | no | Parallel chunk encoding + compression (rayon) |
 | `fast-checksum` | no | crc32fast-accelerated checksums |
 | `lz4` | no | LZ4 block compression filter (id 32004) |
 | `zstd` | no | Zstandard compression filter (id 32015) |
-| `pcodec` | no | Pcodec lossless numerical codec (id 32023, via `pco` crate) |
-| `system-zlib` / `zlib-rs` | no | Alternative zlib backends for deflate |
+| `pcodec` | no | Pcodec lossless numerical codec (via `pco` crate). Private, unregistered filter id 480: **only clawhdf5 can read these datasets** (h5py/libhdf5 cannot). Files from clawhdf5 <= 2.7.0 used id 32023, which is registered to Granular BitRound; they still read. |
+| `system-zlib` | no | System zlib backend for deflate (C) |
 | `blake3_hash` | no | BLAKE3 content hashing for provenance |
+| `szip` | no | SZIP filter (id 4) via libaec (C, through the internal `libaec-sys` crate) |
 
 ### `clawhdf5-ann`
 
 | Flag | Default | Description |
 |------|---------|-------------|
-| `parallel` | no | Rayon-parallel neighbor-distance computation during HNSW graph pruning |
+| `parallel` | no | Batched bulk build runs neighbour planning and back-link pruning on a Rayon pool; the graph is identical with or without it (enabled by `clawhdf5-agent`'s default `parallel`) |
 
 ### `clawhdf5-io`
 
 | Flag | Default | Description |
 |------|---------|-------------|
+| `mmap` | no | Memory-mapped reads (`memmap2`) |
+| `async` | no | Tokio-based async I/O |
+| `hsds` | no | HSDS (HDF REST service) client |
 | `mpi-io` | no | MPI-backed I/O via the `mpi` crate |
 
 > **Parallel I/O (MPI) limitation:** `mpi-io`'s read path is a root-rank read
@@ -494,17 +727,17 @@ back to a scalar loop. See `BENCHMARKS.md`, "Quantising the index copy".
 ## Building
 
 ```bash
-# Default
+# Default (pure Rust: no cmake or C compiler needed)
 cargo build --workspace
 
 # Agent memory with all accelerations (Linux)
-cargo build -p clawhdf5-agent --features "agent,float16,parallel,fast-math"
+cargo build -p clawhdf5-agent --features fast-math
 
 # Agent memory with Apple Accelerate (macOS)
-cargo build -p clawhdf5-agent --features "agent,float16,accelerate,parallel,gpu"
+cargo build -p clawhdf5-agent --features "accelerate,gpu"
 
 # Tests
-cargo test --workspace            # all 1,650+ tests
+cargo test --workspace            # all 1,850+ tests
 cargo test -p clawhdf5-agent      # agent memory tests
 scripts/ci-test.sh                # what CI runs: fmt, clippy matrix, tests,
                                   # h5py/netCDF4 interop, no_std
@@ -526,25 +759,42 @@ cargo bench -p clawhdf5-bench               # h5bench-equivalent I/O suite
 
 ```
 agent_memory.h5
-├── /meta
-│   ├── schema_version: "1.0"
-│   ├── agent_id, embedder, embedding_dim
-│   └── created_at
+├── /meta                          (attributes)
+│   ├── schema_version: "1.0", edgehdf5_version
+│   ├── agent_id, embedder, embedding_dim, chunk_size, overlap, created_at
+│   ├── float16, compression, compression_level, compact_threshold,
+│   │   hebbian_boost, decay_factor, wal_enabled, wal_max_entries
+│   ├── quantized_index, hnsw_m, hnsw_ef_construction, hnsw_ef_search
+│   ├── wal_applied_len, wal_applied_crc   (WAL mark of the last checkpoint)
+│   └── ann_generation                     (ties the .ann sidecar to this checkpoint)
 ├── /memory
 │   ├── chunks:              string[N]
-│   ├── embeddings:  f32[N × D]  (or f16 with float16 flag)
+│   ├── embeddings:          f32[N × D], or f16 for a `float16` store
+│   │                        (chunked; deflate, or Zstd with the `zstd`
+│   │                         feature, when compression is on)
+│   ├── source_channel:      string[N]
+│   ├── timestamps:          f64[N]
+│   ├── session_ids:         string[N]
+│   ├── tags:                string[N]
 │   ├── tombstones:          u8[N]
-│   └── norms:       f32[N]      (pre-computed L2)
+│   ├── norms:               f32[N]      (pre-computed L2)
+│   └── activation_weights:  f32[N]      (Hebbian)
 ├── /sessions
-│   ├── ids:         string[S]
-│   └── summaries:   string[S]
+│   ├── ids, channels, summaries:  string[S]
+│   ├── start_idxs, end_idxs:      i64[S]
+│   └── timestamps:                f64[S]
 └── /knowledge_graph
-    ├── entity_names:    string[E]
-    ├── relation_srcs:   i64[R]
-    ├── relation_tgts:   i64[R]
-    └── relation_types:  string[R]
+    ├── entity_ids, entity_emb_idxs: i64[E]; entity_names, entity_types: string[E]
+    ├── relation_srcs, relation_tgts: i64[R]; relation_types: string[R]
+    ├── relation_weights: f32[R]; relation_ts: f64[R]
+    └── alias_strings: string[A]; alias_entity_ids: i64[A]   (when aliases exist)
 ```
 
+Alongside the store: `<store>.h5.wal` (write-ahead log), `<store>.h5.ann`
+(HNSW graph; derived, safe to delete) and `<store>.h5.lock` (single-writer
+lock). A second writer gets `MemoryError::Locked`; use
+`HDF5Memory::open_read_only` for a lock-free point-in-time view.
+
 ---
 
 ## Migration
@@ -563,9 +813,39 @@ Replace in `Cargo.toml` and source:
 
 ```bash
 cargo install --path crates/clawhdf5-migrate
-clawhdf5-migrate --sqlite old.db --hdf5 memory.h5 --agent-id my-agent --embedding-dim 384
+clawhdf5-migrate --sqlite old.db --hdf5 memory.h5 --agent-id my-agent --embedder minilm
 ```
 
+The output is an ordinary `clawhdf5-agent` store, written through the agent's
+own API: open it with `HDF5Memory::open` (or `clawhdf5-cli --path memory.h5 …`)
+and search it straight away. The source must use the `memory_chunks` / `sessions` / `entities` / `relations` layout (names are
+configurable with `--*-table`); note that this is not ZeroClaw's schema, and
+ZeroClaw does not use clawhdf5. What carries over:
+
+| SQLite | Agent store |
+|--------|-------------|
+| `memory_chunks` | memory records (text, embedding, source channel, timestamp, session id, tags); rows with `deleted = 1` become deleted records, or are left out with `--skip-deleted` |
+| `sessions` | sessions (id, start/end index, channel, summary, timestamp) |
+| `entities`, `relations` | knowledge graph entities and relations; entities get new ids and relations are re-pointed at them |
+
+The chunk `id` column has no counterpart in the agent store, so records are
+written in `id` order and numbered from 0. Embeddings are stored as float16
+like any new store; `--f32` keeps full precision (and is required for values
+beyond ±65504). The embedding dimension is detected from the first row unless
+`--embedding-dim` is given, and every row must have it: a row of another length
+is an error, never truncated or padded. A source with no memory records (only
+sessions or the graph) needs `--embedding-dim`, since a store's dimension is
+fixed when it is created. Every row is checked before the output is created,
+so a source that cannot be migrated leaves an existing store at `--hdf5` as it
+was. `--incremental` adds to an existing store only the rows it does not
+already hold; the source must have the store's dimension, and records already
+in the store take the source's deleted flag (a row deleted in SQLite since the
+last run is deleted in the store; one un-deleted there is written again, as
+the agent has no un-delete). The tool reads the result back with
+`HDF5Memory::open_read_only`, compares it with the source (every row with
+`--validate-full`) and checks that a migrated record is found by search;
+`--dry-run` only counts the rows.
+
 ---
 
 ## Roadmap
@@ -579,10 +859,10 @@ See [ROADMAP.md](ROADMAP.md) for the full implementation tracker.
 - ✅ Temporal reasoning with sub-µs queries
 - ✅ Memory security + anomaly detection
 - ✅ Multi-modal memory (text/image/audio/video)
-- ✅ OpenClaw integration layer
+- ✅ Markdown ingest/export backend (`ClawhdfBackend`); an OpenClaw plugin was never built — see [docs/openclaw.md](docs/openclaw.md)
 - ✅ Comprehensive Criterion benchmarks
 
-**Phase 2** — MemoryArena and LongMemEval academic benchmarks are done (see [BENCHMARKS.md](BENCHMARKS.md), reproduced on a second machine); remaining: publish the OpenClaw TypeScript bridge to npm, crates.io/PyPI publishing.
+**Phase 2** — MemoryArena and LongMemEval academic benchmarks are done (see [BENCHMARKS.md](BENCHMARKS.md), reproduced on a second machine); remaining: crates.io/PyPI publishing. The Node bindings are unpublished and known to be broken ([known issues](docs/known-issues.md)).
 
 ---
 
@@ -599,6 +879,6 @@ MIT
 ---
 
 <p align="center">
-  <em>Built by <a href="https://github.com/redclawsystems">RedClaw Systems</a></em><br>
-  <em>~92,000 lines of Rust. Zero C dependencies. One file to remember everything.</em>
+  <em>Built by <a href="https://git.redclaw.dev/quantumclaw">RedClaw Systems</a></em><br>
+  <em>~86,000 lines of Rust. Zero C dependencies. One file to remember everything.</em>
 </p>

ROADMAP.md

@@ -105,24 +105,30 @@
 
 ---
 
-## Track 7: OpenClaw Integration
-**Status:** 🟢 Complete
+## Track 7: OpenClaw Integration — withdrawn (2026-09-25)
+**Status:** ⚪ Withdrawn (the items below were library work; no OpenClaw integration shipped)
 **Priority:** Critical (for adoption)
 **Crates:** `clawhdf5-agent`, `clawhdf5-napi`
 
 - [x] **7.1** Memory backend trait — MemoryBackend with search/get/write/ingest/export/stats
 - [x] **7.2** Hybrid retrieval pipeline — ClawhdfBackend wires RRF → reranker → confidence rejection
 - [x] **7.3** Markdown import/export — MarkdownParser + MarkdownExporter with line tracking + metadata
-- [x] **7.4** memory_search tool — backed by full hybrid retrieval pipeline
-- [x] **7.5** memory_get tool — get() with path + line range support
+- [x] **7.4** `search()` — backed by the full hybrid retrieval pipeline (a Rust method; no OpenClaw tool was ever registered)
+- [x] **7.5** `get()` — read back by path, with a line slice (not an OpenClaw tool either)
 - [x] **7.6** Compaction integration — run_compaction() (decay + compact + WAL flush), run_consolidation() (hippocampal engine), tick_session(), flush_wal()
-- [x] **7.7** Config surface — `memory.backend = "clawhdf5"` schema documented in docs/openclaw-config.md
-- [x] **7.8** Documentation + migration guide — docs/migration-guide.md, docs/openclaw-integration.md (architecture, full API reference, code patterns)
+- [ ] **7.7** ~~Config surface — `memory.backend = "clawhdf5"`~~ — never valid OpenClaw config; docs removed
+- [ ] **7.8** ~~Documentation + migration guide~~ — removed: they described an integration that never worked
 
-**Node.js bridge:** `clawhdf5-napi` (napi-rs) → `@redclaw/clawhdf5` npm package with full TypeScript types.
+**Node.js bridge:** `clawhdf5-napi` (napi-rs) and a TypeScript wrapper in `packages/clawhdf5-node` exist but are unpublished, untested in CI and known to be broken (docs/known-issues.md).
 
 ---
 
+> **Withdrawn.** None of this track produced a working OpenClaw integration: no
+> plugin was built, the documented `memory.backend = "clawhdf5"` config was never
+> valid in any OpenClaw release, and the Node package was never published. The
+> Rust `ClawhdfBackend` remains as a library API. Not pursued for now; see
+> [docs/openclaw.md](docs/openclaw.md) for what a plugin would need today.
+
 ## Track 8: Benchmarking & Validation
 **Status:** 🟢 Complete
 **Priority:** High
@@ -142,7 +148,7 @@
 
 **Phase 1:** ~~Tracks 1, 2, 3 — core memory intelligence~~ 🟢 Complete
 **Phase 2:** ~~Track 4 (temporal) + Track 5 (security)~~ 🟢 Complete
-**Phase 3:** ~~Track 6 (multi-modal) + Track 7 (OpenClaw integration)~~ 🟢 Complete
+**Phase 3:** ~~Track 6 (multi-modal)~~ 🟢 Complete; Track 7 (OpenClaw integration) withdrawn
 **Phase 4:** ~~Track 8 (benchmarking + validation)~~ 🟢 Complete
 
 All 8 tracks delivered. 1,650+ tests passing, zero clippy warnings.

crates/clawhdf5-accel/Cargo.toml

@@ -2,6 +2,7 @@
 name = "clawhdf5-accel"
 version = "2.7.0"
 edition = "2024"
+rust-version.workspace = true
 description = "SIMD-accelerated operations for rustyhdf5"
 license = "MIT"
 repository = "https://git.redclaw.dev/quantumclaw/clawhdf5"

crates/clawhdf5-accel/src/lib.rs

@@ -124,11 +124,24 @@ pub fn dot_product(a: &[f32], b: &[f32]) -> f32 {
 
 /// Dot product of two `i8` slices, widened to `i32`.
 ///
-/// The kernel behind int8-quantised vector search. Uses the AVX2 path
-/// whenever AVX2 is present — including on AVX-512 machines, where it is
-/// what the f32 kernels use too on a default build.
+/// The kernel behind int8-quantised vector search. On x86-64 it uses the AVX2
+/// path whenever AVX2 is present (including on AVX-512 machines, where it is
+/// what the f32 kernels use too on a default build). On aarch64 it uses the
+/// ARMv8.2 `SDOT` instruction when the CPU has the dot-product extension, and
+/// plain NEON otherwise.
 pub fn dot_i8(a: &[i8], b: &[i8]) -> i32 {
     match detect_backend() {
+        #[cfg(target_arch = "aarch64")]
+        Backend::Neon => {
+            if std::arch::is_aarch64_feature_detected!("dotprod") {
+                // SAFETY: the dotprod extension was just detected at runtime.
+                unsafe { neon::dot_i8_dotprod(a, b) }
+            } else {
+                // SAFETY: NEON is always available on aarch64.
+                unsafe { neon::dot_i8(a, b) }
+            }
+        }
+
         #[cfg(target_arch = "x86_64")]
         // SAFETY: both variants imply AVX2 was detected at runtime (the
         // AVX-512 backend is only selected on CPUs that also have AVX2).
@@ -760,6 +773,42 @@ mod dot_i8_tests {
         }
     }
 
+    /// Dispatch only ever takes one path on a given CPU, so on a machine with
+    /// the dot-product extension the plain-NEON kernel would otherwise go
+    /// untested. Check each aarch64 kernel against scalar directly.
+    #[cfg(target_arch = "aarch64")]
+    #[test]
+    fn every_aarch64_kernel_matches_scalar_exactly() {
+        for len in [0, 1, 7, 15, 16, 17, 31, 32, 33, 63, 64, 100, 384, 385, 1536] {
+            let a = codes(len, 7 + len as u64);
+            let b = codes(len, 7000 + len as u64);
+            let want = scalar::dot_i8(&a, &b);
+            // SAFETY: NEON is always available on aarch64.
+            assert_eq!(unsafe { neon::dot_i8(&a, &b) }, want, "neon, len {len}");
+            if std::arch::is_aarch64_feature_detected!("dotprod") {
+                // SAFETY: the dotprod extension was just detected.
+                assert_eq!(
+                    unsafe { neon::dot_i8_dotprod(&a, &b) },
+                    want,
+                    "dotprod, len {len}"
+                );
+            }
+        }
+        // The extremes, through both kernels.
+        let lo = vec![-128i8; 4096];
+        let hi = vec![127i8; 4096];
+        // SAFETY: NEON is always available on aarch64.
+        assert_eq!(unsafe { neon::dot_i8(&lo, &lo) }, 4096 * 128 * 128);
+        // SAFETY: NEON is always available on aarch64.
+        assert_eq!(unsafe { neon::dot_i8(&lo, &hi) }, -4096 * 128 * 127);
+        if std::arch::is_aarch64_feature_detected!("dotprod") {
+            // SAFETY: the dotprod extension was just detected.
+            assert_eq!(unsafe { neon::dot_i8_dotprod(&lo, &lo) }, 4096 * 128 * 128);
+            // SAFETY: the dotprod extension was just detected.
+            assert_eq!(unsafe { neon::dot_i8_dotprod(&lo, &hi) }, -4096 * 128 * 127);
+        }
+    }
+
     #[test]
     fn extremes_do_not_overflow() {
         // -128 * -128 is the largest product; a long run of it must still fit.

crates/clawhdf5-accel/src/neon.rs

@@ -180,3 +180,130 @@ pub fn checksum_fletcher32(data: &[u8]) -> u32 {
 
     (sum2 << 16) | sum1
 }
+
+/// NEON dot product of two `i8` slices, widened to `i32`, for any aarch64 CPU.
+///
+/// `vmull_s8` multiplies eight lanes into `i16` — even `-128 * -128` is 16 384,
+/// inside `i16` — and `vpadalq_s16` adds adjacent pairs of those into `i32`
+/// accumulators, so nothing can overflow before the final horizontal sum.
+///
+/// CPUs with the ARMv8.2 dot-product extension should use
+/// [`dot_i8_dotprod`], which does the multiply and the accumulate in one
+/// instruction.
+///
+/// # Safety
+/// Caller must ensure aarch64 target (NEON always available).
+// SAFETY: NEON is always available on aarch64 targets; caller guarantees aarch64.
+#[target_feature(enable = "neon")]
+pub unsafe fn dot_i8(a: &[i8], b: &[i8]) -> i32 {
+    assert_eq!(a.len(), b.len());
+    let len = a.len();
+    let mut i = 0;
+    let mut acc0 = vdupq_n_s32(0);
+    let mut acc1 = vdupq_n_s32(0);
+
+    while i + 16 <= len {
+        // SAFETY: NEON is available per the # Safety contract, and both
+        // 16-byte loads start at an index checked against `len` above.
+        unsafe {
+            let va = vld1q_s8(a.as_ptr().add(i));
+            let vb = vld1q_s8(b.as_ptr().add(i));
+            acc0 = vpadalq_s16(acc0, vmull_s8(vget_low_s8(va), vget_low_s8(vb)));
+            acc1 = vpadalq_s16(acc1, vmull_high_s8(va, vb));
+        }
+        i += 16;
+    }
+
+    let mut sum = vaddvq_s32(vaddq_s32(acc0, acc1));
+    while i < len {
+        sum += i32::from(a[i]) * i32::from(b[i]);
+        i += 1;
+    }
+    sum
+}
+
+/// One `SDOT`: for each of the four `i32` lanes of `acc`, add the dot
+/// product of the corresponding four `i8` pairs from `a` and `b`.
+///
+/// Written as inline assembly because the `vdotq_s32` intrinsic is still
+/// behind the unstable `stdarch_neon_dotprod` feature; inline assembly is
+/// stable on aarch64.
+///
+/// # Safety
+/// Caller must ensure the CPU supports the `dotprod` extension.
+#[inline]
+#[target_feature(enable = "neon,dotprod")]
+unsafe fn sdot(acc: int32x4_t, a: int8x16_t, b: int8x16_t) -> int32x4_t {
+    let mut acc = acc;
+    // SAFETY: `dotprod` is enabled for this function and the caller
+    // guarantees the CPU supports it. The instruction reads only its three
+    // vector registers and touches no memory.
+    unsafe {
+        std::arch::asm!(
+            "sdot {acc:v}.4s, {a:v}.16b, {b:v}.16b",
+            acc = inout(vreg) acc,
+            a = in(vreg) a,
+            b = in(vreg) b,
+            options(pure, nomem, nostack),
+        );
+    }
+    acc
+}
+
+/// NEON dot product of two `i8` slices using the ARMv8.2 dot-product
+/// extension (`SDOT`): sixteen multiply-accumulates per instruction, straight
+/// into `i32` lanes.
+///
+/// Present on the cores this crate actually runs on — Cortex-A76 and later
+/// (Raspberry Pi 5, current Android phones), Neoverse-N1 (Graviton2, Ampere
+/// Altra), and every Apple Silicon generation.
+///
+/// # Safety
+/// Caller must verify `is_aarch64_feature_detected!("dotprod")`.
+// SAFETY: caller has verified the dotprod extension at runtime.
+#[target_feature(enable = "neon,dotprod")]
+pub unsafe fn dot_i8_dotprod(a: &[i8], b: &[i8]) -> i32 {
+    assert_eq!(a.len(), b.len());
+    let len = a.len();
+    let mut i = 0;
+    let mut acc0 = vdupq_n_s32(0);
+    let mut acc1 = vdupq_n_s32(0);
+
+    // Two independent accumulators so consecutive SDOTs are not serialised on
+    // one register.
+    while i + 32 <= len {
+        // SAFETY: dotprod is available per the # Safety contract, and every
+        // 16-byte load starts at an index checked against `len` above.
+        unsafe {
+            acc0 = sdot(
+                acc0,
+                vld1q_s8(a.as_ptr().add(i)),
+                vld1q_s8(b.as_ptr().add(i)),
+            );
+            acc1 = sdot(
+                acc1,
+                vld1q_s8(a.as_ptr().add(i + 16)),
+                vld1q_s8(b.as_ptr().add(i + 16)),
+            );
+        }
+        i += 32;
+    }
+    if i + 16 <= len {
+        // SAFETY: as above; the load is bounds-checked by this condition.
+        unsafe {
+            acc0 = sdot(
+                acc0,
+                vld1q_s8(a.as_ptr().add(i)),
+                vld1q_s8(b.as_ptr().add(i)),
+            );
+        }
+        i += 16;
+    }
+
+    let mut sum = vaddvq_s32(vaddq_s32(acc0, acc1));
+    while i < len {
+        sum += i32::from(a[i]) * i32::from(b[i]);
+        i += 1;
+    }
+    sum
+}

crates/clawhdf5-agent/Cargo.toml

@@ -2,6 +2,7 @@
 name = "clawhdf5-agent"
 version = "2.7.0"
 edition = "2024"
+rust-version.workspace = true
 description = "HDF5-backed persistent memory store for on-device AI agents"
 license = "MIT"
 repository = "https://git.redclaw.dev/quantumclaw/clawhdf5"
@@ -18,6 +19,10 @@ clawhdf5-ann = { path = "../clawhdf5-ann", version = "2.7.0", optional = true }
 clawhdf5-gpu = { path = "../clawhdf5-gpu", version = "2.7.0", optional = true, default-features = false }
 serde = { workspace = true }
 byteorder = "1"
+# Signed checkpoints (MemoryConfig-independent; see `signing`). Pure Rust.
+ed25519-dalek = { version = "2", features = ["rand_core"] }
+sha2 = "0.10"
+rand_core = { version = "0.6", features = ["getrandom"] }
 half = { workspace = true, optional = true }
 rayon = { version = "1", optional = true }
 matrixmultiply = { version = "0.3", optional = true }
@@ -44,6 +49,10 @@ harness = false
 name = "memory_bench"
 harness = false
 
+[[bench]]
+name = "multimodal_bench"
+harness = false
+
 [features]
 default = ["float16", "hnsw", "parallel"]
 float16 = ["half"]
@@ -59,7 +68,6 @@ zstd = ["clawhdf5/zstd"]
 # `--no-default-features` (plus re-enabling other defaults) to force the exact
 # linear cosine scan.
 hnsw = ["clawhdf5-ann"]
-agent = []
 gpu = ["clawhdf5-gpu/gpu-wgpu"]
 fast-math = ["matrixmultiply"]
 accelerate = ["accelerate-src", "cblas-sys"]

crates/clawhdf5-agent/benches/multimodal_bench.rs

@@ -0,0 +1,107 @@
+//! Multi-modal memory search benchmarks (`clawhdf5_agent::multimodal`).
+//!
+//! Covers `MultiModalStore::search_cross_modal` (every embedding of every
+//! record, whatever its modality) and, for comparison,
+//! `MultiModalStore::search_by_modality` restricted to one modality.
+//!
+//! Corpus: N records (1K and 10K), each carrying two 384-dim embeddings —
+//! a text embedding of its caption plus one embedding of its primary modality,
+//! cycling Image / Audio / Video — so a cross-modal query scores 2N vectors.
+//! All data comes from a fixed-seed LCG, so every run sees the same corpus.
+//!
+//! Run: `cargo bench -p clawhdf5-agent --bench multimodal_bench`
+
+use std::collections::HashMap;
+
+use clawhdf5_agent::multimodal::{
+    MediaRef, ModalEmbedding, Modality, MultiModalRecord, MultiModalStore,
+};
+use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main};
+
+// ---------------------------------------------------------------------------
+// Simple deterministic PRNG (LCG), same as the other agent benches
+// ---------------------------------------------------------------------------
+
+struct Rng(u32);
+
+impl Rng {
+    fn new(seed: u32) -> Self {
+        Self(seed)
+    }
+    fn next_u32(&mut self) -> u32 {
+        self.0 = self.0.wrapping_mul(1103515245).wrapping_add(12345);
+        self.0 >> 16
+    }
+    fn next_f32(&mut self) -> f32 {
+        self.next_u32() as f32 / 65536.0 - 0.5
+    }
+}
+
+fn make_vec(rng: &mut Rng, dim: usize) -> Vec<f32> {
+    (0..dim).map(|_| rng.next_f32()).collect()
+}
+
+// ---------------------------------------------------------------------------
+// Corpus
+// ---------------------------------------------------------------------------
+
+const DIM: usize = 384;
+const K: usize = 10;
+
+const MEDIA: [(Modality, &str, &str); 3] = [
+    (Modality::Image, "image/png", "clip-vit-base"),
+    (Modality::Audio, "audio/wav", "clap-base"),
+    (Modality::Video, "video/mp4", "xclip-base"),
+];
+
+fn build_store(n: usize, seed: u32) -> MultiModalStore {
+    let mut rng = Rng::new(seed);
+    let mut store = MultiModalStore::new();
+    for i in 0..n {
+        let (modality, mime, model) = &MEDIA[i % MEDIA.len()];
+        let embeddings = vec![
+            ModalEmbedding::new(Modality::Text, make_vec(&mut rng, DIM), "minilm-l6"),
+            ModalEmbedding::new(modality.clone(), make_vec(&mut rng, DIM), *model),
+        ];
+        store.add_record(MultiModalRecord {
+            id: 0,
+            primary_modality: modality.clone(),
+            text_content: Some(format!("{modality} memory {i}")),
+            media_ref: Some(MediaRef::path(format!("/media/{i}"), *mime)),
+            embeddings,
+            observation: None,
+            timestamp: 1_700_000_000.0 + i as f64,
+            metadata: HashMap::new(),
+        });
+    }
+    store
+}
+
+// ---------------------------------------------------------------------------
+// Benchmarks
+// ---------------------------------------------------------------------------
+
+fn multimodal_search_benches(c: &mut Criterion) {
+    let query = make_vec(&mut Rng::new(99), DIM);
+
+    let mut group = c.benchmark_group("multimodal_search");
+    group.sample_size(50);
+
+    for (label, n) in [("1k", 1_000usize), ("10k", 10_000)] {
+        let store = build_store(n, 42);
+        assert_eq!(store.count(), n);
+
+        group.bench_with_input(BenchmarkId::new("cross_modal", label), &n, |b, _| {
+            b.iter(|| store.search_cross_modal(&query, K));
+        });
+
+        group.bench_with_input(BenchmarkId::new("by_modality_image", label), &n, |b, _| {
+            b.iter(|| store.search_by_modality(&Modality::Image, &query, K));
+        });
+    }
+
+    group.finish();
+}
+
+criterion_group!(multimodal_benches, multimodal_search_benches);
+criterion_main!(multimodal_benches);

crates/clawhdf5-agent/src/cache.rs

@@ -1,6 +1,7 @@
 //! In-memory cache for memory entries, sessions, and knowledge graph.
 
 use crate::vector_search;
+use clawhdf5_format::float16::round_to_f16;
 
 /// Every entry's embedding, in one contiguous `[N x dim]` buffer.
 ///
@@ -149,6 +150,11 @@ pub struct MemoryCache {
     pub norms: Vec<f32>,
     /// Hebbian activation weights (default 1.0 per entry).
     pub activation_weights: Vec<f32>,
+    /// Round every embedding to IEEE half precision as it enters the cache,
+    /// so the cache holds exactly what a `float16` store writes to disk. Set
+    /// it with [`MemoryCache::set_half_precision`], which also rounds the
+    /// rows already held.
+    pub half_precision: bool,
 }
 
 impl MemoryCache {
@@ -164,9 +170,44 @@ impl MemoryCache {
             embedding_dim,
             norms: Vec::new(),
             activation_weights: Vec::new(),
+            half_precision: false,
         }
     }
 
+    /// Switch half-precision rounding on or off. Turning it on rounds every
+    /// embedding already held (and recomputes norms where one changed) —
+    /// e.g. a `float16` store whose last checkpoint predates half-precision
+    /// storage and so is still `f32` on disk.
+    pub fn set_half_precision(&mut self, on: bool) {
+        self.half_precision = on;
+        if !on {
+            return;
+        }
+        for i in 0..self.embeddings.len() {
+            let row = &self.embeddings[i];
+            if row
+                .iter()
+                .all(|&v| round_to_f16(v).to_bits() == v.to_bits())
+            {
+                continue;
+            }
+            let rounded: Vec<f32> = row.iter().map(|&v| round_to_f16(v)).collect();
+            self.norms[i] = vector_search::compute_norm(&rounded);
+            self.embeddings.set(i, &rounded);
+        }
+    }
+
+    /// The embedding as the cache will hold it: rounded to half precision
+    /// when [`Self::half_precision`] is on, otherwise unchanged.
+    fn stored_form(&self, mut embedding: Vec<f32>) -> Vec<f32> {
+        if self.half_precision {
+            for v in &mut embedding {
+                *v = round_to_f16(*v);
+            }
+        }
+        embedding
+    }
+
     /// Kept for callers that used to have to re-flatten after a bulk load.
     /// The buffer is always flat now, so there is nothing to rebuild.
     #[deprecated(note = "embeddings are stored flat; this is a no-op")]
@@ -202,6 +243,7 @@ impl MemoryCache {
         tags: String,
     ) -> usize {
         let idx = self.chunks.len();
+        let embedding = self.stored_form(embedding);
         let norm = vector_search::compute_norm(&embedding);
         self.chunks.push(chunk);
         self.embeddings.push(&embedding);
@@ -240,6 +282,7 @@ impl MemoryCache {
         session_id: String,
     ) {
         if idx < self.chunks.len() {
+            let embedding = self.stored_form(embedding);
             let norm = vector_search::compute_norm(&embedding);
             self.chunks[idx] = chunk;
             self.embeddings.set(idx, &embedding);
@@ -439,4 +482,60 @@ mod tests {
             .reset_from(2, vec![vec![1.0, 2.0], vec![3.0, 4.0]]);
         assert_eq!(cache.embeddings.as_flat(), vec![1.0, 2.0, 3.0, 4.0]);
     }
+
+    #[test]
+    fn set_half_precision_rounds_existing_rows_and_their_norms() {
+        // A store with float16 set whose checkpoint is still f32 on disk
+        // loads full-precision rows; switching rounding on must bring them to
+        // exactly what the next checkpoint will write.
+        let mut cache = MemoryCache::new(3);
+        cache.push(
+            "a".into(),
+            vec![0.1, 0.2, 0.3],
+            "c".into(),
+            0.0,
+            "s".into(),
+            "".into(),
+        );
+        cache.push(
+            "b".into(),
+            vec![0.5, 0.25, 1.0],
+            "c".into(),
+            0.0,
+            "s".into(),
+            "".into(),
+        );
+        let exact_norm = cache.norms[0];
+
+        cache.set_half_precision(true);
+        let row0: Vec<f32> = [0.1f32, 0.2, 0.3]
+            .iter()
+            .map(|&v| round_to_f16(v))
+            .collect();
+        assert_eq!(&cache.embeddings[0], row0.as_slice());
+        assert_eq!(cache.norms[0], vector_search::compute_norm(&row0));
+        assert_ne!(cache.norms[0], exact_norm);
+        // Already representable: untouched.
+        assert_eq!(&cache.embeddings[1], &[0.5, 0.25, 1.0]);
+
+        // New rows are rounded as they arrive, and updates too.
+        cache.push(
+            "c".into(),
+            vec![0.1, 0.0, 0.0],
+            "c".into(),
+            0.0,
+            "s".into(),
+            "".into(),
+        );
+        assert_eq!(cache.embeddings[2][0], round_to_f16(0.1));
+        cache.update(
+            2,
+            "c".into(),
+            vec![0.3, 0.0, 0.0],
+            "c".into(),
+            0.0,
+            "s".into(),
+        );
+        assert_eq!(cache.embeddings[2][0], round_to_f16(0.3));
+    }
 }

crates/clawhdf5-agent/src/consolidation.rs

@@ -144,9 +144,44 @@ pub struct ConsolidationStats {
 
 pub struct ImportanceScorer;
 
+/// Sum of squares, in 8-wide lanes so it vectorises.
+fn sum_of_squares(a: &[f32]) -> f32 {
+    let (blocks, tail) = a.as_chunks::<8>();
+    let mut acc = [0.0f32; 8];
+    for b in blocks {
+        for i in 0..8 {
+            acc[i] += b[i] * b[i];
+        }
+    }
+    acc.iter().sum::<f32>() + tail.iter().map(|x| x * x).sum::<f32>()
+}
+
+/// `(a · b, |b|²)` in one pass over equal-length slices, in 8-wide lanes.
+fn dot_and_norm2(a: &[f32], b: &[f32]) -> (f32, f32) {
+    let (a_blocks, a_tail) = a.as_chunks::<8>();
+    let (b_blocks, b_tail) = b.as_chunks::<8>();
+    let mut dot = [0.0f32; 8];
+    let mut nb = [0.0f32; 8];
+    for (x, y) in a_blocks.iter().zip(b_blocks) {
+        for i in 0..8 {
+            dot[i] += x[i] * y[i];
+            nb[i] += y[i] * y[i];
+        }
+    }
+    let mut d = dot.iter().sum::<f32>();
+    let mut n = nb.iter().sum::<f32>();
+    for (x, y) in a_tail.iter().zip(b_tail) {
+        d += x * y;
+        n += y * y;
+    }
+    (d, n)
+}
+
 impl ImportanceScorer {
     /// Cosine similarity between two embedding slices.
-    /// Returns 0.0 if either norm is zero.
+    /// Returns 0.0 if either norm is zero. The reference that
+    /// [`Self::score_surprise`] is tested against.
+    #[cfg(test)]
     fn cosine_similarity(a: &[f32], b: &[f32]) -> f32 {
         let len = a.len().min(b.len());
         if len == 0 {
@@ -167,13 +202,54 @@ impl ImportanceScorer {
 
     /// Novelty score: 1.0 − max cosine similarity against all existing records.
     /// Returns 1.0 when there are no existing memories.
+    ///
+    /// Same result as the reference cosine similarity against each record, but the
+    /// new embedding's norm is computed once rather than per record, each
+    /// record costs one fused pass (dot product and its norm together) rather
+    /// than three, and a large working set is scored in parallel. Every insert
+    /// scores against the whole working tier, so this is what an unbounded
+    /// working tier pays for: at 100K records it was the difference between a
+    /// benchmark finishing and not (`BENCHMARKS.md`, "Consolidation Efficiency").
     pub fn score_surprise(embedding: &[f32], existing_memories: &[&MemoryRecord]) -> f32 {
         if existing_memories.is_empty() {
             return 1.0;
         }
+        let query_norm2 = sum_of_squares(embedding);
+        let similarity = |r: &&MemoryRecord| -> f32 {
+            let other = &r.embedding;
+            let len = embedding.len().min(other.len());
+            if len == 0 {
+                return 0.0;
+            }
+            let (dot, other_norm2) = dot_and_norm2(&embedding[..len], &other[..len]);
+            // A shorter record compares against the query's matching prefix.
+            let q2 = if len == embedding.len() {
+                query_norm2
+            } else {
+                sum_of_squares(&embedding[..len])
+            };
+            if q2 == 0.0 || other_norm2 == 0.0 {
+                return 0.0;
+            }
+            dot / (q2.sqrt() * other_norm2.sqrt())
+        };
+        #[cfg(feature = "parallel")]
+        let max_sim = if existing_memories.len() >= 4096 {
+            use rayon::prelude::*;
+            existing_memories
+                .par_iter()
+                .map(similarity)
+                .reduce(|| f32::NEG_INFINITY, f32::max)
+        } else {
+            existing_memories
+                .iter()
+                .map(similarity)
+                .fold(f32::NEG_INFINITY, f32::max)
+        };
+        #[cfg(not(feature = "parallel"))]
         let max_sim = existing_memories
             .iter()
-            .map(|r| Self::cosine_similarity(embedding, &r.embedding))
+            .map(similarity)
             .fold(f32::NEG_INFINITY, f32::max);
         (1.0 - max_sim).clamp(0.0, 1.0)
     }
@@ -471,6 +547,54 @@ impl ConsolidationEngine {
 mod tests {
     use super::*;
 
+    #[test]
+    fn score_surprise_matches_the_reference_cosine() {
+        let mut x = 0x2545_F491_4F6C_DD1Du64;
+        let mut next = || {
+            x ^= x << 13;
+            x ^= x >> 7;
+            x ^= x << 17;
+            (x >> 40) as f32 / (1u64 << 24) as f32 - 0.5
+        };
+        let make = |id: u64, v: Vec<f32>| MemoryRecord {
+            id,
+            chunk: String::new(),
+            embedding: v,
+            tier: MemoryTier::Working,
+            importance: 0.0,
+            access_count: 0,
+            last_accessed: 0.0,
+            created_at: 0.0,
+            source: MemorySource::User,
+        };
+        // Ordinary rows, a shorter one, an empty one and a zero vector; and
+        // enough rows to take the parallel path too.
+        for n in [5usize, 5000] {
+            let mut recs: Vec<MemoryRecord> = (0..n as u64)
+                .map(|i| make(i, (0..37).map(|_| next()).collect()))
+                .collect();
+            recs.push(make(9_000, (0..20).map(|_| next()).collect()));
+            recs.push(make(9_001, Vec::new()));
+            recs.push(make(9_002, vec![0.0; 37]));
+            let refs: Vec<&MemoryRecord> = recs.iter().collect();
+            for _ in 0..5 {
+                let q: Vec<f32> = (0..37).map(|_| next()).collect();
+                let expected = (1.0
+                    - refs
+                        .iter()
+                        .map(|r| ImportanceScorer::cosine_similarity(&q, &r.embedding))
+                        .fold(f32::NEG_INFINITY, f32::max))
+                .clamp(0.0, 1.0);
+                let got = ImportanceScorer::score_surprise(&q, &refs);
+                assert!((got - expected).abs() < 1e-5, "n={n}: {got} vs {expected}");
+            }
+        }
+        assert_eq!(
+            ImportanceScorer::score_surprise(&[0.0; 4], &[&make(1, vec![1.0; 4])]),
+            1.0
+        );
+    }
+
     // Helper: build a simple normalised embedding of given dimension.
     fn unit_vec(dim: usize, hot: usize) -> Vec<f32> {
         let mut v = vec![0.0f32; dim];

crates/clawhdf5-agent/src/hybrid.rs

@@ -65,31 +65,34 @@ pub fn hybrid_search_fused(
 ) -> Vec<(usize, f32)> {
     // Get raw scores from both systems. Request all results so normalization
     // covers the full distribution.
-    // Use parallel search when rayon feature is enabled and vector count > 10K.
-    let vec_scores = {
-        #[cfg(feature = "parallel")]
-        {
-            if vectors.count() > 10_000 {
-                vector_search::parallel_cosine_batch(
-                    query_embedding,
-                    vectors,
-                    tombstones,
-                    vectors.count(),
-                )
-            } else {
-                vector_search::cosine_similarity_batch(query_embedding, vectors, tombstones)
-            }
-        }
-        #[cfg(not(feature = "parallel"))]
-        {
-            vector_search::cosine_similarity_batch(query_embedding, vectors, tombstones)
-        }
-    };
+    let vec_scores = exact_vector_scores(query_embedding, vectors, tombstones);
     let kw_scores = bm25_index.scores(query_text);
 
     fuse(vec_scores, kw_scores, fusion, k)
 }
 
+/// Cosine similarity of `query_embedding` to every vector whose `skip` byte is
+/// 0 (a tombstone, or any other exclusion mask). Parallel above 10K vectors
+/// when the `parallel` feature is on.
+pub fn exact_vector_scores(
+    query_embedding: &[f32],
+    vectors: &(impl crate::vector_search::VectorSet + Sync + ?Sized),
+    skip: &[u8],
+) -> Vec<(usize, f32)> {
+    #[cfg(feature = "parallel")]
+    {
+        if vectors.count() > 10_000 {
+            return vector_search::parallel_cosine_batch(
+                query_embedding,
+                vectors,
+                skip,
+                vectors.count(),
+            );
+        }
+    }
+    vector_search::cosine_similarity_batch(query_embedding, vectors, skip)
+}
+
 /// Merge pre-computed vector-similarity and keyword scores into a single ranking.
 ///
 /// Both score sets are independently min-max normalized to [0, 1] and combined

crates/clawhdf5-agent/src/knowledge.rs

@@ -163,12 +163,13 @@ fn levenshtein(a: &str, b: &str) -> usize {
 /// entities-slice-index map, and an entity-id -> relation-indices map (edges
 /// touching that entity as either source or target).
 ///
-/// Built fresh per traversal call rather than cached on `KnowledgeCache`:
-/// entities/relations are plain `pub` `Vec`s that get pushed to directly
-/// (e.g. `schema.rs`'s load path bypasses `add_entity`/`add_relation`), so a
-/// persistent index would need extra bookkeeping to avoid drifting stale. A
-/// one-off O(V+E) build per call is still a large win over the O(V·E) (BFS)
-/// / O(steps·active·E) (spreading activation) scans it replaces.
+/// Cached on `KnowledgeCache` and checked against a fingerprint of the graph
+/// on every use ([`graph_fingerprint`]). entities/relations are plain `pub`
+/// `Vec`s that get changed directly (e.g. `schema.rs`'s load path bypasses
+/// `add_entity`/`add_relation`), so the cache cannot rely on being told about
+/// changes; the fingerprint notices any of them. Rebuilding it on every
+/// traversal instead made a 2-hop BFS over 1K entities 6.5x slower than the
+/// scan it replaced (24 -> 155 µs; `BENCHMARKS.md`, "Knowledge Graph").
 struct AdjacencyIndex {
     entity_index: HashMap<u64, usize>,
     by_entity: HashMap<u64, Vec<usize>>,
@@ -204,6 +205,45 @@ impl AdjacencyIndex {
     }
 }
 
+/// A hash of everything [`AdjacencyIndex`] depends on — each entity's id and
+/// position, each relation's endpoints and position. One linear pass, no
+/// allocation: far cheaper than building the index, which hashes the same
+/// values into two maps.
+fn graph_fingerprint(entities: &[Entity], relations: &[Relation]) -> u64 {
+    // splitmix64-style mixing; order matters, so positions are covered.
+    fn mix(h: u64, v: u64) -> u64 {
+        let mut z = (h ^ v).wrapping_add(0x9E37_79B9_7F4A_7C15);
+        z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
+        z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
+        z ^ (z >> 31)
+    }
+    let mut h = mix(entities.len() as u64, relations.len() as u64);
+    for e in entities {
+        h = mix(h, e.id);
+    }
+    for r in relations {
+        h = mix(mix(h, r.src), r.tgt);
+    }
+    h
+}
+
+/// The cached [`AdjacencyIndex`] and the fingerprint it was built for.
+/// Cloning a `KnowledgeCache` starts the clone with an empty cache.
+#[derive(Default)]
+struct AdjacencyCache(std::sync::Mutex<Option<(u64, std::sync::Arc<AdjacencyIndex>)>>);
+
+impl Clone for AdjacencyCache {
+    fn clone(&self) -> Self {
+        Self::default()
+    }
+}
+
+impl std::fmt::Debug for AdjacencyCache {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.write_str("AdjacencyCache")
+    }
+}
+
 // ---------------------------------------------------------------------------
 // KnowledgeCache
 // ---------------------------------------------------------------------------
@@ -216,6 +256,7 @@ pub struct KnowledgeCache {
     pub alias_strings: Vec<String>,
     pub alias_entity_ids: Vec<i64>,
     next_entity_id: u64,
+    adjacency: AdjacencyCache,
 }
 
 impl KnowledgeCache {
@@ -226,6 +267,7 @@ impl KnowledgeCache {
             alias_strings: Vec::new(),
             alias_entity_ids: Vec::new(),
             next_entity_id: 0,
+            adjacency: AdjacencyCache::default(),
         }
     }
 
@@ -236,9 +278,29 @@ impl KnowledgeCache {
             alias_strings: Vec::new(),
             alias_entity_ids: Vec::new(),
             next_entity_id: next_id,
+            adjacency: AdjacencyCache::default(),
         }
     }
 
+    /// The adjacency index for the graph as it is now: the cached one if the
+    /// graph's fingerprint still matches, otherwise rebuilt and cached.
+    fn adjacency_index(&self) -> std::sync::Arc<AdjacencyIndex> {
+        let fp = graph_fingerprint(&self.entities, &self.relations);
+        let mut slot = self
+            .adjacency
+            .0
+            .lock()
+            .unwrap_or_else(std::sync::PoisonError::into_inner);
+        if let Some((cached_fp, idx)) = slot.as_ref()
+            && *cached_fp == fp
+        {
+            return idx.clone();
+        }
+        let idx = std::sync::Arc::new(AdjacencyIndex::build(&self.entities, &self.relations));
+        *slot = Some((fp, idx.clone()));
+        idx
+    }
+
     // -----------------------------------------------------------------------
     // Entity management
     // -----------------------------------------------------------------------
@@ -397,7 +459,7 @@ impl KnowledgeCache {
     /// together with their discovered depth.  The seed entity itself is NOT
     /// included.  Traversal follows both outgoing and incoming relation edges.
     pub fn bfs_neighbors(&self, entity_id: u64, max_depth: usize) -> Vec<(Entity, usize)> {
-        let idx = AdjacencyIndex::build(&self.entities, &self.relations);
+        let idx = self.adjacency_index();
         let mut visited: HashSet<u64> = HashSet::new();
         let mut queue: VecDeque<(u64, usize)> = VecDeque::new();
         let mut results: Vec<(Entity, usize)> = Vec::new();
@@ -502,7 +564,7 @@ impl KnowledgeCache {
         min_activation: f32,
         max_steps: usize,
     ) -> Vec<(u64, f32)> {
-        let idx = AdjacencyIndex::build(&self.entities, &self.relations);
+        let idx = self.adjacency_index();
         let mut activation: HashMap<u64, f32> = HashMap::new();
 
         // Initialise seeds with activation 1.0.
@@ -631,6 +693,51 @@ impl Default for KnowledgeCache {
 mod tests {
     use super::*;
 
+    #[test]
+    fn cached_adjacency_sees_direct_changes_to_the_graph() {
+        // The index is cached across traversals, but entities/relations are
+        // pub Vecs anyone can edit; every kind of edit must be seen.
+        let mut kg = KnowledgeCache::new();
+        let a = kg.add_entity("a", "t", -1);
+        let b = kg.add_entity("b", "t", -1);
+        let c = kg.add_entity("c", "t", -1);
+        kg.add_relation(a, b, "r", 1.0);
+        let ids = |kg: &KnowledgeCache| -> Vec<u64> {
+            let mut v: Vec<u64> = kg.bfs_neighbors(a, 3).iter().map(|(e, _)| e.id).collect();
+            v.sort();
+            v
+        };
+        assert_eq!(ids(&kg), vec![b]);
+        assert_eq!(ids(&kg), vec![b], "cached index reused");
+
+        // Pushed directly, bypassing add_relation.
+        kg.relations.push(Relation {
+            src: b,
+            tgt: c,
+            ..Relation::default()
+        });
+        assert_eq!(ids(&kg), vec![b, c]);
+
+        // Rewired in place: same lengths, different edge.
+        kg.relations[1].tgt = a;
+        assert_eq!(ids(&kg), vec![b]);
+
+        // Removed and replaced: same lengths again.
+        kg.relations.pop();
+        kg.relations.push(Relation {
+            src: a,
+            tgt: c,
+            ..Relation::default()
+        });
+        assert_eq!(ids(&kg), vec![b, c]);
+        let act: Vec<u64> = kg
+            .spreading_activation(&[a], 0.5, 0.0, 2)
+            .iter()
+            .map(|(id, _)| *id)
+            .collect();
+        assert!(act.contains(&c));
+    }
+
     // -----------------------------------------------------------------------
     // Original tests — must remain passing
     // -----------------------------------------------------------------------

crates/clawhdf5-agent/src/lib.rs

@@ -1,4 +1,4 @@
-//! ZeroClaw agent memory HDF5 backend.
+//! Agent memory stored in a single HDF5 file.
 //!
 //! Provides persistent memory storage for AI agents using HDF5 files.
 //! All data is cached in-memory for fast access and flushed to disk
@@ -36,6 +36,7 @@ pub mod reranker;
 pub mod schema;
 pub mod search;
 pub mod session;
+pub mod signing;
 pub mod storage;
 mod store_lock;
 pub mod temporal;
@@ -63,6 +64,7 @@ use std::path::{Path, PathBuf};
 use cache::MemoryCache;
 #[cfg(feature = "hnsw")]
 use clawhdf5_ann::{DistanceMetric, HnswIndex, Storage};
+use clawhdf5_format::float16::round_to_f16;
 use ephemeral::{EphemeralConfig, EphemeralStore};
 
 // EphemeralEntry and EphemeralStats are part of the crate public API via
@@ -71,11 +73,13 @@ use ephemeral::{EphemeralConfig, EphemeralStore};
 pub use ephemeral::{EphemeralEntry, EphemeralStats};
 use knowledge::KnowledgeCache;
 use memory_strategy::{Exchange, MemoryStrategy, StrategyOutput};
-use session::SessionCache;
+pub use search::SearchOptions;
+pub use session::{SessionCache, SessionEntry};
 
 // --- Error type ---
 
 #[derive(Debug)]
+#[non_exhaustive]
 pub enum MemoryError {
     Io(std::io::Error),
     Hdf5(String),
@@ -83,6 +87,14 @@ pub enum MemoryError {
     NotFound(String),
     /// Another `HDF5Memory` (in this or another process) has the store open.
     Locked(String),
+    /// A record the store cannot hold as given, e.g. an embedding value
+    /// outside the half-precision range of a `float16` store.
+    InvalidEntry(String),
+    /// The store's checkpoints are signed and no signing key is set, so a
+    /// checkpoint would leave it unsigned. Set the key with
+    /// [`HDF5Memory::set_signing_key`], or drop the signature on purpose with
+    /// [`HDF5Memory::remove_signature`].
+    SigningKeyRequired(String),
 }
 
 impl std::fmt::Display for MemoryError {
@@ -93,6 +105,8 @@ impl std::fmt::Display for MemoryError {
             MemoryError::Schema(e) => write!(f, "schema error: {e}"),
             MemoryError::NotFound(e) => write!(f, "not found: {e}"),
             MemoryError::Locked(e) => write!(f, "store is locked: {e}"),
+            MemoryError::InvalidEntry(e) => write!(f, "invalid entry: {e}"),
+            MemoryError::SigningKeyRequired(e) => write!(f, "signing key required: {e}"),
         }
     }
 }
@@ -124,6 +138,19 @@ pub struct MemoryConfig {
     pub embedding_dim: usize,
     pub chunk_size: usize,
     pub overlap: usize,
+    /// Store embeddings as IEEE half precision (numpy `float16`): half the
+    /// bytes of the embeddings dataset on disk. Every embedding is rounded to
+    /// the nearest half as it enters the store, in memory as well as on disk,
+    /// so search results are the same before and after a reopen. Values must
+    /// lie within ±65504; a save outside that is `MemoryError::InvalidEntry`.
+    /// Fixed when the store is created (persisted in `/meta`).
+    ///
+    /// **On by default for new stores**: on the full LongMemEval haystack with
+    /// real MiniLM embeddings every retrieval metric matched `f32`, and at
+    /// 100K records the file is 48% smaller (`BENCHMARKS.md`). Existing
+    /// stores keep the setting they were created with. Set it to `false` for
+    /// full-precision embeddings, e.g. for unnormalised vectors that may
+    /// exceed the half-precision range.
     pub float16: bool,
     pub compression: bool,
     pub compression_level: u32,
@@ -134,13 +161,19 @@ pub struct MemoryConfig {
     pub wal_enabled: bool,
     pub wal_max_entries: usize,
     /// Store the vector index's own copy of the embeddings as int8 rather than
-    /// f32, a quarter of the memory.
+    /// f32, a quarter of the memory. **On by default** for new stores.
     ///
     /// The index's copy is the single largest part of a loaded store's
     /// footprint. Quantised distances are approximate, so the candidate pool
     /// is re-scored against the cache's exact embeddings before fusion, which
-    /// restores recall; what it costs is throughput — roughly 13% of queries
-    /// per second and 16% of build time at 100K x 384. See `BENCHMARKS.md`.
+    /// holds recall at the f32 index's level. It is also faster, not slower:
+    /// at equal recall, 1.63x the queries per second on x86-64 (AVX2) and
+    /// 1.18x on a Raspberry Pi 5 (NEON `SDOT`), with builds 1.8x and 2.3x
+    /// faster. See `BENCHMARKS.md`.
+    ///
+    /// Persisted with the store. Stores written before this setting existed
+    /// have no stored value and open as `false`, so reopening an old store
+    /// never changes how its index is held.
     ///
     /// Has no effect without the `hnsw` feature.
     pub quantized_index: bool,
@@ -173,7 +206,7 @@ impl MemoryConfig {
             embedding_dim,
             chunk_size: 512,
             overlap: 50,
-            float16: false,
+            float16: true,
             compression: false,
             compression_level: 0,
             compact_threshold: 0.3,
@@ -182,7 +215,7 @@ impl MemoryConfig {
             created_at,
             wal_enabled: true,
             wal_max_entries: 500,
-            quantized_index: false,
+            quantized_index: true,
             hnsw_m: 16,
             hnsw_ef_construction: 64,
             hnsw_ef_search: 0,
@@ -292,6 +325,12 @@ pub struct HDF5Memory {
     activations_dirty: bool,
     /// Opened with [`HDF5Memory::open_read_only`]: nothing may reach the disk.
     read_only: bool,
+    /// Key that signs every checkpoint; never persisted. See
+    /// [`HDF5Memory::set_signing_key`].
+    signing_key: Option<signing::SigningKey>,
+    /// Checkpoints of this store are signed: the file on disk is, or a key
+    /// has been set. A checkpoint without a key is then refused.
+    signed: bool,
     /// A WAL that `open()` could not read and moved aside; see
     /// [`HDF5Memory::quarantined_wal`].
     quarantined_wal: Option<PathBuf>,
@@ -311,7 +350,8 @@ impl HDF5Memory {
     /// Create a new HDF5 memory file with the given configuration.
     pub fn create(config: MemoryConfig) -> Result<Self> {
         let lock = store_lock::StoreLock::acquire(&config.path)?;
-        let cache = MemoryCache::new(config.embedding_dim);
+        let mut cache = MemoryCache::new(config.embedding_dim);
+        cache.set_half_precision(config.float16);
         let sessions = SessionCache::new();
         let knowledge = KnowledgeCache::new();
 
@@ -346,6 +386,8 @@ impl HDF5Memory {
             bm25_filter: bm25::TokenFilter::default(),
             activations_dirty: false,
             read_only: false,
+            signing_key: None,
+            signed: false,
             quarantined_wal: None,
             _lock: Some(lock),
         })
@@ -524,6 +566,8 @@ impl HDF5Memory {
             bm25_filter: bm25::TokenFilter::default(),
             activations_dirty: false,
             read_only,
+            signing_key: None,
+            signed: checkpoint.signed,
             quarantined_wal,
             _lock: lock,
         })
@@ -684,6 +728,39 @@ impl HDF5Memory {
         }
     }
 
+    /// Sign every checkpoint from now on with `key` (Ed25519). The key is
+    /// never written anywhere; set it again after every `open`. Once a store
+    /// is signed, a checkpoint without the key is refused
+    /// ([`MemoryError::SigningKeyRequired`]) rather than silently leaving it
+    /// unsigned. Setting a different key re-signs the store under that key
+    /// from the next checkpoint; a verifier trusting the old key will then
+    /// reject it, which is the point. Call [`AgentMemory::flush_wal`] to sign
+    /// right away.
+    pub fn set_signing_key(&mut self, key: signing::SigningKey) {
+        self.signing_key = Some(key);
+        self.signed = true;
+    }
+
+    /// Stop signing: the next checkpoint writes the store unsigned. The
+    /// deliberate way out of [`MemoryError::SigningKeyRequired`].
+    pub fn remove_signature(&mut self) {
+        self.signing_key = None;
+        self.signed = false;
+    }
+
+    /// Checkpoints of this store are signed (on disk, or from the next
+    /// checkpoint because a key has been set).
+    pub fn is_signed(&self) -> bool {
+        self.signed
+    }
+
+    /// Check the checkpoint at `path` against the public key the caller
+    /// trusts; see [`signing::verify_store`]. Reads the file only: it works
+    /// on a store another process has open.
+    pub fn verify(path: &Path, trusted: &signing::VerifyingKey) -> Result<signing::VerifyReport> {
+        signing::verify_store(path, trusted)
+    }
+
     /// Flush current state to disk and truncate the WAL.
     ///
     /// Every code path that persists the full cache to the .h5 file must
@@ -699,10 +776,28 @@ impl HDF5Memory {
         // Record which WAL prefix this checkpoint contains, so a crash before
         // the truncate below can't replay those entries a second time.
         let wal_applied = self.wal.as_ref().map(|w| w.mark());
+        let signature = match &self.signing_key {
+            Some(key) => Some(signing::sign(
+                key,
+                &self.config,
+                &self.cache,
+                &self.sessions,
+                &self.knowledge,
+                wal_applied,
+            )),
+            None if self.signed => {
+                return Err(MemoryError::SigningKeyRequired(format!(
+                    "{} is signed; set its signing key before a checkpoint \
+                     (saves so far are held in the WAL or in memory)",
+                    self.config.path.display()
+                )));
+            }
+            None => None,
+        };
         // Written before the .h5 so a crash in between leaves a sidecar whose
         // generation matches no checkpoint (ignored), never the reverse.
         let ann_generation = self.persist_vector_index();
-        storage::write_to_disk_with_meta(
+        storage::write_to_disk_signed(
             &self.config.path,
             &self.config,
             &self.cache,
@@ -711,7 +806,9 @@ impl HDF5Memory {
             &schema::CheckpointMeta {
                 wal_applied,
                 ann_generation,
+                signed: signature.is_some(),
             },
+            signature.as_ref(),
         )?;
         if let Some(ref mut w) = self.wal {
             w.truncate()?;
@@ -1002,6 +1099,18 @@ impl HDF5Memory {
         &self.config
     }
 
+    /// The sessions recorded in this store.
+    pub fn sessions(&self) -> &SessionCache {
+        &self.sessions
+    }
+
+    /// Mutable access to the sessions, e.g. to add many at once. Changes
+    /// reach the disk at the next checkpoint (any flushing call, such as
+    /// [`HDF5Memory::flush_wal`] or `save_batch`), not immediately.
+    pub fn sessions_mut(&mut self) -> &mut SessionCache {
+        &mut self.sessions
+    }
+
     /// Get a reference to the knowledge cache.
     pub fn knowledge(&self) -> &KnowledgeCache {
         &self.knowledge
@@ -1064,7 +1173,29 @@ impl HDF5Memory {
     /// Upsert: if an active entry with the same tags (key) exists, update it in-place.
     /// Otherwise append a new entry. Use this for key-based memory stores where
     /// the same key should not create duplicates.
+    /// A `float16` store holds embeddings as IEEE half precision, which has no
+    /// finite value beyond ±65504. Refuse such an embedding rather than
+    /// silently store infinity. (Values that are already infinite or NaN are
+    /// stored as they are, as in an `f32` store.)
+    fn check_embedding(&self, embedding: &[f32]) -> Result<()> {
+        if !self.config.float16 {
+            return Ok(());
+        }
+        let overflow = embedding
+            .iter()
+            .enumerate()
+            .find(|&(_, &v)| v.is_finite() && round_to_f16(v).is_infinite());
+        match overflow {
+            None => Ok(()),
+            Some((i, v)) => Err(MemoryError::InvalidEntry(format!(
+                "embedding[{i}] = {v} is outside the half-precision range (±65504) \
+                 of this float16 store"
+            ))),
+        }
+    }
+
     pub fn save_or_update(&mut self, entry: MemoryEntry) -> Result<usize> {
+        self.check_embedding(&entry.embedding)?;
         if let Some(existing_idx) = self.cache.find_by_tags(&entry.tags) {
             if let Some(ref mut w) = self.wal {
                 let wal_entry = wal::WalEntry {
@@ -1120,6 +1251,7 @@ impl HDF5Memory {
 
 impl AgentMemory for HDF5Memory {
     fn save(&mut self, entry: MemoryEntry) -> Result<usize> {
+        self.check_embedding(&entry.embedding)?;
         if let Some(ref mut w) = self.wal {
             let wal_entry = wal::WalEntry {
                 entry_type: wal::WalEntryType::Save,
@@ -1161,6 +1293,10 @@ impl AgentMemory for HDF5Memory {
     }
 
     fn save_batch(&mut self, entries: Vec<MemoryEntry>) -> Result<Vec<usize>> {
+        // All or nothing: check every entry before storing any.
+        for entry in &entries {
+            self.check_embedding(&entry.embedding)?;
+        }
         let mut indices = Vec::with_capacity(entries.len());
         for entry in entries {
             let idx = self.cache.push(
@@ -1321,6 +1457,9 @@ impl HDF5Memory {
         })?;
         let view = memory_strategy::CacheStoreView::new(&self.cache, &self.knowledge);
         let output = strat.evaluate(&exchange, &view);
+        for e in &output.entries {
+            self.check_embedding(&e.embedding)?;
+        }
         for e in &output.entries {
             self.cache.push(
                 e.chunk.clone(),
@@ -1345,6 +1484,35 @@ impl HDF5Memory {
 }
 
 impl HDF5Memory {
+    /// Delete many records with a single checkpoint, where
+    /// [`AgentMemory::delete`] checkpoints once per record.
+    ///
+    /// All or nothing: if any id is out of range or already deleted (or
+    /// repeated), nothing is deleted and `MemoryError::NotFound` is returned.
+    /// Unlike `delete`, this never auto-compacts, so the records stay in the
+    /// store as tombstones (their indices unchanged) until [`AgentMemory::compact`]
+    /// is called — importers use it to carry over records that were already
+    /// deleted in the source.
+    pub fn delete_batch(&mut self, ids: &[usize]) -> Result<()> {
+        let mut seen = std::collections::HashSet::with_capacity(ids.len());
+        for &id in ids {
+            if self.cache.tombstones.get(id).copied() != Some(0) || !seen.insert(id) {
+                return Err(MemoryError::NotFound(format!(
+                    "entry {id} not found or already deleted"
+                )));
+            }
+        }
+        if ids.is_empty() {
+            return Ok(());
+        }
+        for &id in ids {
+            self.cache.mark_deleted(id);
+            self.hnsw_on_delete(id);
+            self.bm25_on_delete(id);
+        }
+        self.flush()
+    }
+
     pub fn tick_session(&mut self) -> Result<()> {
         let d = self.config.decay_factor;
         for w in self.cache.activation_weights.iter_mut() {
@@ -1405,6 +1573,16 @@ impl HDF5Memory {
         let mut promoted = 0;
 
         for key in candidates {
+            // Check before taking, so a rejected entry stays in the ephemeral
+            // tier rather than being lost.
+            if let Some(emb) = self
+                .ephemeral
+                .as_ref()
+                .and_then(|s| s.get_entry(&key))
+                .and_then(|e| e.embedding.as_deref())
+            {
+                self.check_embedding(emb)?;
+            }
             let entry = match self
                 .ephemeral
                 .as_mut()
@@ -1533,6 +1711,79 @@ mod tests {
         }
     }
 
+    #[test]
+    fn delete_batch_tombstones_without_compacting() {
+        let dir = TempDir::new().unwrap();
+        let path = dir.path().join("test.h5");
+        let mut mem = HDF5Memory::create(make_config(&dir)).unwrap();
+        mem.save_batch(
+            (0..4)
+                .map(|i| make_entry(&format!("record {i}"), &[i as f32, 1.0, 0.0, 0.0]))
+                .collect(),
+        )
+        .unwrap();
+        // 3 of 4 is far past compact_threshold (0.3): delete() would compact.
+        mem.delete_batch(&[0, 1, 3]).unwrap();
+        assert_eq!(mem.count(), 4);
+        assert_eq!(mem.count_active(), 1);
+        drop(mem);
+
+        let mut mem = HDF5Memory::open(&path).unwrap();
+        assert_eq!(mem.cache.tombstones, vec![1, 1, 0, 1]);
+        let hits = mem.hybrid_search(&[0.0, 1.0, 0.0, 0.0], "record", 0.5, 0.5, 10);
+        assert!(
+            hits.iter().all(|r| r.index == 2),
+            "tombstoned record returned"
+        );
+    }
+
+    #[test]
+    fn delete_batch_is_all_or_nothing() {
+        let dir = TempDir::new().unwrap();
+        let mut mem = HDF5Memory::create(make_config(&dir)).unwrap();
+        mem.save_batch(vec![
+            make_entry("a", &[1.0, 0.0, 0.0, 0.0]),
+            make_entry("b", &[0.0, 1.0, 0.0, 0.0]),
+        ])
+        .unwrap();
+        for bad in [&[0, 5][..], &[1, 1][..]] {
+            assert!(matches!(
+                mem.delete_batch(bad),
+                Err(MemoryError::NotFound(_))
+            ));
+            assert_eq!(mem.count_active(), 2, "{bad:?} deleted something");
+        }
+        mem.delete_batch(&[]).unwrap();
+        assert_eq!(mem.count_active(), 2);
+    }
+
+    #[test]
+    fn sessions_mut_add_at_keeps_timestamp_across_reopen() {
+        let dir = TempDir::new().unwrap();
+        let path = dir.path().join("test.h5");
+        let mut mem = HDF5Memory::create(make_config(&dir)).unwrap();
+        mem.sessions_mut()
+            .add_at("s-old", 2, 7, "discord", "old summary", 1.7e15);
+        mem.flush_wal().unwrap();
+        drop(mem);
+
+        let mem = HDF5Memory::open_read_only(&path).unwrap();
+        let s = mem.sessions();
+        assert_eq!(s.len(), 1);
+        let e = &s.entries[0];
+        assert_eq!(
+            (
+                e.id.as_str(),
+                e.start_idx,
+                e.end_idx,
+                e.channel.as_str(),
+                e.ts
+            ),
+            ("s-old", 2, 7, "discord", 1.7e15)
+        );
+        assert_eq!(s.summaries[0], "old summary");
+    }
+
     #[test]
     fn create_new_file() {
         let dir = TempDir::new().unwrap();

crates/clawhdf5-agent/src/openclaw.rs

@@ -1,10 +1,12 @@
-//! OpenClaw Integration Layer.
+//! A Markdown-oriented memory backend over [`crate::HDF5Memory`].
 //!
-//! Bridge between OpenClaw agent gateway (Markdown + sqlite-vec) and the
-//! clawhdf5 HDF5-backed memory backend.  Provides:
+//! Named for OpenClaw, whose workspace memory is Markdown, but **not an
+//! OpenClaw plugin**: nothing here registers with OpenClaw, and the
+//! integration it was written for never worked (see `docs/openclaw.md`).
+//! Provides:
 //!
-//! - [`MemoryBackend`] — the trait OpenClaw implements against.
-//! - [`ClawhdfBackend`] — concrete HDF5-backed implementation.
+//! - [`MemoryBackend`] — search / read back / write / ingest / export.
+//! - [`ClawhdfBackend`] — the HDF5-backed implementation.
 //! - [`MarkdownParser`] — splits Markdown into [`MarkdownSection`] records.
 //! - [`MarkdownExporter`] — renders sections back to Markdown text.
 
@@ -13,9 +15,8 @@ use std::path::{Path, PathBuf};
 use std::time::{SystemTime, UNIX_EPOCH};
 
 use crate::{
-    AgentMemory, HDF5Memory, MemoryConfig, MemoryEntry,
-    confidence::{ConfidenceConfig, ScoredResult, reject_low_confidence},
-    reranker::{ReRankConfig, RerankInput, rerank},
+    AgentMemory, HDF5Memory, MemoryConfig, MemoryEntry, SearchOptions,
+    confidence::ConfidenceConfig, reranker::ReRankConfig,
 };
 
 // ─────────────────────────────────────────────────────────────────────────────
@@ -62,7 +63,8 @@ pub struct BackendStats {
 // MemoryBackend trait
 // ─────────────────────────────────────────────────────────────────────────────
 
-/// Interface that OpenClaw uses to interact with a memory backend.
+/// A Markdown-oriented memory backend: search, read back by path, write,
+/// ingest and export.
 ///
 /// Implementors provide persistent storage, full-text + vector search,
 /// Markdown ingestion / export, and statistics.
@@ -319,7 +321,7 @@ impl MarkdownExporter {
 ///
 /// # Path mapping
 ///
-/// OpenClaw addresses memories by file path (e.g. `"memory/user.md"`).
+/// Memories are addressed by file path (e.g. `"memory/user.md"`).
 /// Internally every [`MemoryEntry`] stores the originating path as its
 /// `source_channel`.  Section sub-paths are stored as
 /// `"<path>::<heading>"`.
@@ -422,7 +424,7 @@ impl ClawhdfBackend {
 
     // ── Compaction & Consolidation hooks (7.6) ────────────────────────────
 
-    /// Run a compaction cycle — called by OpenClaw during session compaction.
+    /// Run a compaction cycle (decay, compaction, WAL flush).
     ///
     /// Sequence:
     /// 1. `tick_session()` — apply Hebbian decay to all activation weights.
@@ -524,71 +526,27 @@ impl ClawhdfBackend {
 
 impl MemoryBackend for ClawhdfBackend {
     /// Search using hybrid vector + BM25 retrieval, then re-rank and
-    /// confidence-filter.
+    /// confidence-filter — [`HDF5Memory::search`] with both stages on.
     fn search(
         &mut self,
         query_text: &str,
         query_embedding: &[f32],
         k: usize,
     ) -> Vec<MemorySearchResult> {
-        // 1. Hybrid retrieval (vector + BM25, fused by score).
-        let candidates = k.saturating_mul(3).max(10);
-        let raw = self.memory.hybrid_search_with(
-            query_embedding,
-            query_text,
-            crate::hybrid::DEFAULT_FUSION,
-            candidates,
-        );
-
-        if raw.is_empty() {
-            return Vec::new();
-        }
-
-        let now = Self::now_secs();
-
-        // 2. Re-rank using temporal recency, source authority, Hebbian weight.
-        let rerank_inputs: Vec<RerankInput> = raw
-            .iter()
-            .map(|r| RerankInput {
-                index: r.index,
-                timestamp: r.timestamp,
-                source_channel: r.source_channel.clone(),
-                raw_activation: r.activation,
-                relevance: r.score,
-            })
-            .collect();
-
-        let reranked = rerank(&rerank_inputs, &self.rerank_config, now);
-
-        // 3. Confidence rejection.
-        let scored: Vec<ScoredResult> = reranked
-            .iter()
-            .map(|r| ScoredResult {
-                index: r.index,
-                score: r.combined_score,
-            })
-            .collect();
-
-        let confident = reject_low_confidence(&scored, &self.confidence_config);
-
-        // 4. Map back to MemorySearchResult; preserve raw text via index lookup.
-        let raw_by_idx: HashMap<usize, &crate::SearchResult> =
-            raw.iter().map(|r| (r.index, r)).collect();
-
-        confident
+        let options = SearchOptions::new(k)
+            .with_rerank(self.rerank_config)
+            .with_confidence(self.confidence_config.clone())
+            .at_time(Self::now_secs());
+        self.memory
+            .search(query_embedding, query_text, &options)
             .into_iter()
-            .take(k)
-            .filter_map(|sr| {
-                let r = raw_by_idx.get(&sr.index)?;
-                let path = r.source_channel.clone();
-                Some(MemorySearchResult {
-                    text: r.chunk.clone(),
-                    score: sr.score,
-                    path: path.clone(),
+            .map(|r| MemorySearchResult {
+                text: r.chunk,
+                score: r.score,
+                path: r.source_channel.clone(),
                 line_range: None,
                 timestamp: Some(r.timestamp),
-                    source: path,
-                })
+                source: r.source_channel,
             })
             .collect()
     }

crates/clawhdf5-agent/src/provenance.rs

@@ -2,7 +2,7 @@
 //!
 //! Records the origin, authorship, and a content hash of every memory chunk
 //! so the system can detect *accidental* corruption and trace data lineage.
-//! The hash is unkeyed (see [`fnv1a_64`]) — this is not a tamper-evidence or
+//! The hash is unkeyed (FNV-1a) — this is not a tamper-evidence or
 //! authenticity guarantee.
 
 use std::collections::HashMap;

crates/clawhdf5-agent/src/schema.rs

@@ -15,6 +15,9 @@ use crate::session::SessionCache;
 use crate::wal::WalMark;
 
 pub const SCHEMA_VERSION: &str = "1.0";
+/// Writer-version tag stored in `/meta` as `edgehdf5_version`. Kept for file
+/// compatibility; despite the name it has nothing to do with ZeroClaw, which
+/// does not use clawhdf5.
 pub const ZEROCLAW_VERSION: &str = "0.8.0";
 
 /// `/meta` attributes holding the [`WalMark`] of the WAL prefix already folded
@@ -23,6 +26,7 @@ pub const ZEROCLAW_VERSION: &str = "0.8.0";
 const WAL_APPLIED_LEN_ATTR: &str = "wal_applied_len";
 const WAL_APPLIED_CRC_ATTR: &str = "wal_applied_crc";
 const ANN_GENERATION_ATTR: &str = "ann_generation";
+const SIG_VERSION_ATTR: &str = "sig_version";
 
 /// Build a complete HDF5 file from the in-memory state.
 pub fn build_hdf5_file(
@@ -46,7 +50,7 @@ pub fn build_hdf5_file_with_mark(
 ) -> Result<Vec<u8>, MemoryError> {
     let meta = CheckpointMeta {
         wal_applied,
-        ann_generation: None,
+        ..CheckpointMeta::default()
     };
     build_hdf5_file_with_meta(config, cache, sessions, knowledge, &meta)
 }
@@ -61,6 +65,10 @@ pub struct CheckpointMeta {
     /// one left over from another checkpoint can never be attached to records
     /// it wasn't built from.
     pub ann_generation: Option<u64>,
+    /// The checkpoint carries an Ed25519 signature (see [`crate::signing`]).
+    /// Read-only: whether a checkpoint is *written* signed is decided by the
+    /// signature passed to [`build_hdf5_file_signed`].
+    pub signed: bool,
 }
 
 /// [`build_hdf5_file`] with checkpoint bookkeeping.
@@ -70,6 +78,19 @@ pub fn build_hdf5_file_with_meta(
     sessions: &SessionCache,
     knowledge: &KnowledgeCache,
     checkpoint: &CheckpointMeta,
+) -> Result<Vec<u8>, MemoryError> {
+    build_hdf5_file_signed(config, cache, sessions, knowledge, checkpoint, None)
+}
+
+/// [`build_hdf5_file_with_meta`], plus a signed manifest of the contents
+/// (see [`crate::signing`]).
+pub fn build_hdf5_file_signed(
+    config: &MemoryConfig,
+    cache: &MemoryCache,
+    sessions: &SessionCache,
+    knowledge: &KnowledgeCache,
+    checkpoint: &CheckpointMeta,
+    signature: Option<&crate::signing::StoredSignature>,
 ) -> Result<Vec<u8>, MemoryError> {
     let wal_applied = checkpoint.wal_applied;
     let mut builder = clawhdf5::FileBuilder::new();
@@ -130,11 +151,42 @@ pub fn build_hdf5_file_with_meta(
         // round trip through every reader.
         meta.set_attr(ANN_GENERATION_ATTR, AttrValue::I64(generation as i64));
     }
+    if let Some(sig) = signature {
+        use crate::signing::to_hex;
+        let m = &sig.manifest;
+        meta.set_attr(
+            SIG_VERSION_ATTR,
+            AttrValue::I64(crate::signing::MANIFEST_VERSION),
+        );
+        meta.set_attr("sig_algorithm", AttrValue::String("ed25519".into()));
+        meta.set_attr("sig_public_key", AttrValue::String(to_hex(&sig.public_key)));
+        meta.set_attr("sig_signature", AttrValue::String(to_hex(&sig.signature)));
+        meta.set_attr("sig_record_count", AttrValue::I64(m.record_count as i64));
+        meta.set_attr(
+            "sig_records_root",
+            AttrValue::String(to_hex(&m.records_root)),
+        );
+        meta.set_attr("sig_settings", AttrValue::String(to_hex(&m.settings)));
+        meta.set_attr("sig_sessions", AttrValue::String(to_hex(&m.sessions)));
+        meta.set_attr("sig_graph", AttrValue::String(to_hex(&m.graph)));
+    }
     // Need at least one dataset in the group for it to be a proper group
     meta.create_dataset("_marker").with_u8_data(&[1]).compact();
     let finished_meta = meta.finish();
     builder.add_group(finished_meta);
 
+    // /integrity: the signed per-record hashes, so verification can say
+    // which records changed.
+    if let Some(sig) = signature {
+        let mut group = builder.create_group("integrity");
+        let flat: Vec<u8> = sig.record_hashes.iter().flatten().copied().collect();
+        group
+            .create_dataset("record_hashes")
+            .with_u8_data(&flat)
+            .with_shape(&[sig.record_hashes.len() as u64, 32]);
+        builder.add_group(group.finish());
+    }
+
     // /memory group
     build_memory_group(&mut builder, config, cache)?;
 
@@ -159,20 +211,27 @@ fn build_memory_group(
     // chunks: fixed-length string array
     write_string_dataset(&mut group, "chunks", &cache.chunks);
 
-    // embeddings: f32 [N x D]
+    // embeddings: [N x D], f32 — or IEEE half precision for a `float16`
+    // store. The cache already holds half-rounded values then, so this
+    // conversion is exact and a reopened store sees the same numbers.
     let n = cache.embeddings.len() as u64;
     let d = cache.embedding_dim as u64;
     let flat = cache.flat_embeddings();
     {
-        let ds = group
-            .create_dataset("embeddings")
-            .with_f32_data(flat)
-            .with_shape(&[n, d]);
+        let ds = group.create_dataset("embeddings");
+        let elem_bytes: u64 = if config.float16 {
+            ds.with_f16_data(flat);
+            2
+        } else {
+            ds.with_f32_data(flat);
+            4
+        };
+        ds.with_shape(&[n, d]);
 
         // Chunk size tuning: target ~256KB per chunk for optimal I/O
         if n > 0 && d > 0 {
             let target_chunk_bytes: u64 = 256 * 1024;
-            let rows_per_chunk = (target_chunk_bytes / (d * 4)).max(1).min(n);
+            let rows_per_chunk = (target_chunk_bytes / (d * elem_bytes)).max(1).min(n);
             ds.with_chunks(&[rows_per_chunk, d]);
 
             // Compression. Shuffle is applied automatically (auto-shuffle
@@ -433,6 +492,64 @@ pub fn read_wal_mark(file: &clawhdf5::File) -> Option<WalMark> {
     Some(WalMark { len, crc })
 }
 
+/// Read a checkpoint's signature, if it has one. A signature whose
+/// attributes are present but malformed is an error, not "unsigned".
+pub fn read_signature(
+    file: &clawhdf5::File,
+) -> Result<Option<crate::signing::StoredSignature>, MemoryError> {
+    use crate::signing::{Manifest, StoredSignature, from_hex};
+    let attrs = file
+        .group("meta")
+        .and_then(|g| g.attrs())
+        .map_err(|e| MemoryError::Schema(format!("cannot read /meta attrs: {e}")))?;
+    let version = match attrs.get(SIG_VERSION_ATTR) {
+        None => return Ok(None),
+        Some(AttrValue::I64(v)) => *v,
+        Some(_) => return Err(MemoryError::Schema("malformed sig_version".into())),
+    };
+    if version != crate::signing::MANIFEST_VERSION {
+        return Err(MemoryError::Schema(format!(
+            "unsupported signature version {version}"
+        )));
+    }
+    fn hex<const N: usize>(
+        attrs: &std::collections::HashMap<String, AttrValue>,
+        name: &str,
+    ) -> Result<[u8; N], MemoryError> {
+        match attrs.get(name) {
+            Some(AttrValue::String(s)) => from_hex::<N>(s),
+            _ => None,
+        }
+        .ok_or_else(|| MemoryError::Schema(format!("malformed or missing {name}")))
+    }
+    let record_count = match attrs.get("sig_record_count") {
+        Some(AttrValue::I64(v)) if *v >= 0 => *v as u64,
+        _ => return Err(MemoryError::Schema("malformed sig_record_count".into())),
+    };
+    let group = file
+        .group("integrity")
+        .map_err(|e| MemoryError::Schema(format!("signed checkpoint without /integrity: {e}")))?;
+    let flat = read_u8_dataset(&group, "record_hashes")?;
+    if flat.len() % 32 != 0 {
+        return Err(MemoryError::Schema(
+            "/integrity/record_hashes is not a whole number of hashes".into(),
+        ));
+    }
+    let record_hashes = flat.as_chunks::<32>().0.to_vec();
+    Ok(Some(StoredSignature {
+        manifest: Manifest {
+            record_count,
+            records_root: hex::<32>(&attrs, "sig_records_root")?,
+            settings: hex::<32>(&attrs, "sig_settings")?,
+            sessions: hex::<32>(&attrs, "sig_sessions")?,
+            graph: hex::<32>(&attrs, "sig_graph")?,
+        },
+        record_hashes,
+        public_key: hex::<32>(&attrs, "sig_public_key")?,
+        signature: hex::<64>(&attrs, "sig_signature")?,
+    }))
+}
+
 /// Read the checkpoint bookkeeping from `/meta`.
 pub fn read_checkpoint_meta(file: &clawhdf5::File) -> CheckpointMeta {
     let ann_generation = file
@@ -443,9 +560,14 @@ pub fn read_checkpoint_meta(file: &clawhdf5::File) -> CheckpointMeta {
             Some(AttrValue::I64(v)) => Some(*v as u64),
             _ => None,
         });
+    let signed = file
+        .group("meta")
+        .and_then(|g| g.attrs())
+        .is_ok_and(|attrs| attrs.contains_key(SIG_VERSION_ATTR));
     CheckpointMeta {
         wal_applied: read_wal_mark(file),
         ann_generation,
+        signed,
     }
 }
 
@@ -497,6 +619,9 @@ pub fn validate_and_load(
         wal_max_entries: optional_i64_attr(&attrs, "wal_max_entries")
             .and_then(|v| usize::try_from(v).ok())
             .unwrap_or(500),
+        // `false`, not the new-store default: a store written before this
+        // setting existed was built with an f32 index, and reopening it must
+        // not silently change that.
         quantized_index: optional_bool_attr(&attrs, "quantized_index", false),
         hnsw_m: optional_i64_attr(&attrs, "hnsw_m")
             .and_then(|v| usize::try_from(v).ok())
@@ -510,7 +635,16 @@ pub fn validate_and_load(
     };
 
     // Load /memory group
-    let memory_cache = load_memory_group(file, embedding_dim)?;
+    let mut memory_cache = load_memory_group(file, embedding_dim)?;
+    // A float16 store's cache holds half-rounded embeddings. Embeddings read
+    // from an f16 dataset already are; a float16 store whose last checkpoint
+    // predates half-precision storage is still f32 on disk and is rounded
+    // here.
+    if config.float16 && embeddings_are_f16(file) {
+        memory_cache.half_precision = true;
+    } else {
+        memory_cache.set_half_precision(config.float16);
+    }
 
     // Load /sessions group
     let session_cache = load_sessions_group(file)?;
@@ -753,6 +887,13 @@ fn read_string_dataset_from_group(
         .map_err(|e| MemoryError::Hdf5(format!("cannot read strings from {name}: {e}")))
 }
 
+/// Whether `/memory/embeddings` is stored as IEEE half precision.
+fn embeddings_are_f16(file: &clawhdf5::File) -> bool {
+    file.dataset("memory/embeddings")
+        .and_then(|ds| ds.dtype())
+        .is_ok_and(|dt| matches!(dt, clawhdf5::DType::Other(ref s) if s == "float16"))
+}
+
 fn read_f32_dataset(group: &clawhdf5::Group<'_>, name: &str) -> Result<Vec<f32>, MemoryError> {
     let ds = group
         .dataset(name)

crates/clawhdf5-agent/src/search.rs

@@ -2,18 +2,107 @@
 
 use std::path::Path;
 
+use std::collections::HashSet;
+
 use crate::bm25;
+use crate::confidence::{ConfidenceConfig, ScoredResult, reject_low_confidence};
 use crate::hybrid;
+use crate::reranker::{ReRankConfig, RerankInput, rerank};
 use crate::{HDF5Memory, MAX_ACTIVATION_WEIGHT, MemoryError, Result, SearchResult};
 
+/// Options for [`HDF5Memory::search`].
+///
+/// [`SearchOptions::new`] is plain hybrid search with the tuned default
+/// fusion — the same as `hybrid_search_with(.., hybrid::DEFAULT_FUSION, k)`.
+/// Every stage beyond that is opt-in.
+#[derive(Debug, Clone)]
+pub struct SearchOptions {
+    /// Number of results to return.
+    pub k: usize,
+    /// How the vector and keyword stages are combined.
+    pub fusion: hybrid::Fusion,
+    /// Only consider records whose `source_channel` is one of these. The
+    /// filter applies *before* ranking, so a filtered search still returns up
+    /// to `k` results and scores are normalised over the records it can
+    /// return. `None` searches everything; an empty list matches nothing.
+    pub source_channels: Option<Vec<String>>,
+    /// Re-rank a candidate pool by retrieval relevance, recency, source
+    /// authority and activation — the pipeline the OpenClaw backend runs.
+    pub rerank: Option<ReRankConfig>,
+    /// Candidates retrieved for re-ranking; 0 means `max(3k, 10)`.
+    pub rerank_pool: usize,
+    /// Drop low-confidence results (after re-ranking, when that is on).
+    pub confidence: Option<ConfidenceConfig>,
+    /// The time recency is measured from, in seconds since the epoch.
+    /// `None` uses the system clock.
+    pub now: Option<f64>,
+}
+
+impl SearchOptions {
+    pub fn new(k: usize) -> Self {
+        Self {
+            k,
+            fusion: hybrid::DEFAULT_FUSION,
+            source_channels: None,
+            rerank: None,
+            rerank_pool: 0,
+            confidence: None,
+            now: None,
+        }
+    }
+
+    pub fn with_fusion(mut self, fusion: hybrid::Fusion) -> Self {
+        self.fusion = fusion;
+        self
+    }
+
+    /// Search only records from these source channels.
+    pub fn with_sources<S: Into<String>>(mut self, channels: impl IntoIterator<Item = S>) -> Self {
+        self.source_channels = Some(channels.into_iter().map(Into::into).collect());
+        self
+    }
+
+    pub fn with_rerank(mut self, config: ReRankConfig) -> Self {
+        self.rerank = Some(config);
+        self
+    }
+
+    pub fn with_confidence(mut self, config: ConfidenceConfig) -> Self {
+        self.confidence = Some(config);
+        self
+    }
+
+    /// Measure recency from `now` (seconds since the epoch) instead of the
+    /// system clock — for reproducible results and tests.
+    pub fn at_time(mut self, now: f64) -> Self {
+        self.now = Some(now);
+        self
+    }
+}
+
+impl Default for SearchOptions {
+    fn default() -> Self {
+        Self::new(10)
+    }
+}
+
 impl HDF5Memory {
-    /// Vector + keyword scoring stage of [`HDF5Memory::hybrid_search`].
+    /// Vector + keyword scoring stage of [`HDF5Memory::search`].
     ///
     /// Without the `hnsw` feature this is a full linear cosine scan (the exact
     /// previous behaviour, also used as the correctness oracle in tests). With
     /// `hnsw` enabled and an index available, the vector candidates come from an
     /// approximate-nearest-neighbour search over an over-fetched pool, then merge
     /// with BM25 via the shared [`hybrid::merge_vector_keyword`].
+    ///
+    /// `exclude`, when given, marks records that must not be returned (1 =
+    /// excluded; it covers tombstones too). The index is over-fetched in
+    /// proportion to how much the mask removes. Surfacing `pool` candidates
+    /// costs the index roughly `pool × M` distance evaluations, while an exact
+    /// scan of the allowed records costs one each — so whenever that scan is
+    /// the cheaper of the two it is used instead, and it is also the fallback
+    /// if the pool comes back with too few allowed hits (the allowed records
+    /// sit away from the query). A filtered search never comes back short.
     #[cfg(feature = "hnsw")]
     fn vector_keyword_search(
         &mut self,
@@ -22,16 +111,30 @@ impl HDF5Memory {
         bm25: &bm25::BM25Index,
         fusion: hybrid::Fusion,
         k: usize,
+        exclude: Option<&[u8]>,
     ) -> Vec<(usize, f32)> {
         self.ensure_hnsw_fresh();
+        let n = self.cache.len();
+        // Over-fetch so the merge sees a useful vector pool. `ef` is
+        // configurable, but the pool the fusion stage sees is not tied to it:
+        // a caller lowering `ef` for speed should not silently narrow what
+        // fusion has to work with.
+        let mut pool = (k * 8).max(64);
+        let mut allowed = n;
+        if let Some(ex) = exclude {
+            allowed = ex.iter().filter(|&&e| e == 0).count();
+            if allowed == 0 {
+                return Vec::new();
+            }
+            // Expect `pool` allowed hits if the filter is independent of the
+            // query's neighbourhood.
+            pool = pool.saturating_mul(n).div_ceil(allowed);
+            if allowed <= pool.saturating_mul(self.hnsw_m()) {
+                return self.exact_masked_search(query_embedding, query_text, bm25, fusion, k, ex);
+            }
+        }
         match self.hnsw.as_ref() {
             Some(index) if !index.is_empty() && index.dimension() == query_embedding.len() => {
-                // Over-fetch so the merge sees a useful vector pool; cosine
-                // distance from the index converts back to similarity (1 - d).
-                // `ef` is configurable, but the pool the fusion stage sees is
-                // not tied to it: a caller lowering `ef` for speed should not
-                // silently narrow what fusion has to work with.
-                let pool = (k * 8).max(64);
                 let ef = self.hnsw_ef_search(k).max(pool);
                 let candidates = index.search(query_embedding, pool, ef);
                 // A quantised index returns approximate distances, and no
@@ -42,6 +145,7 @@ impl HDF5Memory {
                 let exact = index.storage() == clawhdf5_ann::Storage::Int8;
                 let vec_scores: Vec<(usize, f32)> = candidates
                     .into_iter()
+                    .filter(|(id, _)| exclude.is_none_or(|ex| ex[*id] == 0))
                     .map(|(id, dist)| {
                         let score = if exact {
                             crate::vector_search::cosine_similarity(
@@ -56,10 +160,54 @@ impl HDF5Memory {
                     .collect();
                 // Fusion normalises over every keyword match, so it needs all
                 // the scores — but not ranked.
-                let kw_scores = bm25.scores(query_text);
+                let mut kw_scores = bm25.scores(query_text);
+                if let Some(ex) = exclude {
+                    if vec_scores.len() < k.min(allowed) {
+                        // The allowed records are not where the index looked.
+                        return self.exact_masked_search(
+                            query_embedding,
+                            query_text,
+                            bm25,
+                            fusion,
+                            k,
+                            ex,
+                        );
+                    }
+                    kw_scores.retain(|(id, _)| ex[*id] == 0);
+                }
                 hybrid::fuse(vec_scores, kw_scores, fusion, k)
             }
-            _ => hybrid::hybrid_search_fused(
+            _ => match exclude {
+                Some(ex) => {
+                    self.exact_masked_search(query_embedding, query_text, bm25, fusion, k, ex)
+                }
+                None => hybrid::hybrid_search_fused(
+                    query_embedding,
+                    query_text,
+                    &self.cache.embeddings,
+                    &self.cache.chunks,
+                    &self.cache.tombstones,
+                    bm25,
+                    fusion,
+                    k,
+                ),
+            },
+        }
+    }
+
+    #[cfg(not(feature = "hnsw"))]
+    fn vector_keyword_search(
+        &mut self,
+        query_embedding: &[f32],
+        query_text: &str,
+        bm25: &bm25::BM25Index,
+        fusion: hybrid::Fusion,
+        k: usize,
+        exclude: Option<&[u8]>,
+    ) -> Vec<(usize, f32)> {
+        match exclude {
+            Some(ex) => self.exact_masked_search(query_embedding, query_text, bm25, fusion, k, ex),
+            None => hybrid::hybrid_search_fused(
                 query_embedding,
                 query_text,
                 &self.cache.embeddings,
@@ -72,25 +220,33 @@ impl HDF5Memory {
         }
     }
 
-    #[cfg(not(feature = "hnsw"))]
-    fn vector_keyword_search(
-        &mut self,
+    /// Exact hybrid search over the records `exclude` leaves (0 = allowed).
+    fn exact_masked_search(
+        &self,
         query_embedding: &[f32],
         query_text: &str,
         bm25: &bm25::BM25Index,
         fusion: hybrid::Fusion,
         k: usize,
+        exclude: &[u8],
     ) -> Vec<(usize, f32)> {
-        hybrid::hybrid_search_fused(
-            query_embedding,
-            query_text,
-            &self.cache.embeddings,
-            &self.cache.chunks,
-            &self.cache.tombstones,
-            bm25,
-            fusion,
-            k,
-        )
+        let vec_scores =
+            hybrid::exact_vector_scores(query_embedding, &self.cache.embeddings, exclude);
+        let mut kw_scores = bm25.scores(query_text);
+        kw_scores.retain(|(id, _)| exclude.get(*id) == Some(&0));
+        hybrid::fuse(vec_scores, kw_scores, fusion, k)
+    }
+
+    /// The exclusion mask for a source-channel filter: 1 for a tombstoned
+    /// record or one from a channel not in `channels`.
+    fn source_mask(&self, channels: &[String]) -> Vec<u8> {
+        let allowed: HashSet<&str> = channels.iter().map(String::as_str).collect();
+        self.cache
+            .source_channels
+            .iter()
+            .zip(&self.cache.tombstones)
+            .map(|(ch, &t)| u8::from(t != 0 || !allowed.contains(ch.as_str())))
+            .collect()
     }
 
     /// Perform hybrid search combining cosine vector similarity and BM25 keyword search.
@@ -125,12 +281,53 @@ impl HDF5Memory {
         fusion: hybrid::Fusion,
         k: usize,
     ) -> Vec<SearchResult> {
+        self.search(
+            query_embedding,
+            query_text,
+            &SearchOptions::new(k).with_fusion(fusion),
+        )
+    }
+
+    /// Hybrid search with optional source filtering, re-ranking and
+    /// confidence rejection — see [`SearchOptions`].
+    ///
+    /// Stages, in order: vector + keyword retrieval over the records the
+    /// source filter allows; fusion; scaling by Hebbian activation; re-ranking
+    /// (if on) of a `rerank_pool` of candidates; confidence rejection (if on);
+    /// the top `k`. The records returned with a positive score get their
+    /// Hebbian boost.
+    pub fn search(
+        &mut self,
+        query_embedding: &[f32],
+        query_text: &str,
+        options: &SearchOptions,
+    ) -> Vec<SearchResult> {
+        let k = options.k;
+        let fetch = match options.rerank {
+            Some(_) if options.rerank_pool > 0 => options.rerank_pool.max(k),
+            Some(_) => k.saturating_mul(3).max(10),
+            None => k,
+        };
+        let exclude = options
+            .source_channels
+            .as_deref()
+            .map(|channels| self.source_mask(channels));
+
         // The keyword index lives for the life of the store and is updated
         // incrementally. Take it out for the duration of the call so the
         // vector stage can borrow `self` mutably, then put it back.
         self.ensure_bm25_fresh();
         let bm25 = self.bm25.take().expect("ensure_bm25_fresh leaves an index");
-        let scored = self.vector_keyword_search(query_embedding, query_text, &bm25, fusion, k);
+        let scored = self.vector_keyword_search(
+            query_embedding,
+            query_text,
+            &bm25,
+            options.fusion,
+            fetch,
+            exclude.as_deref(),
+        );
+        self.bm25 = Some(bm25);
+
         let mut results: Vec<SearchResult> = scored
             .into_iter()
             .map(|(idx, score)| {
@@ -154,6 +351,25 @@ impl HDF5Memory {
                 .then(a.index.cmp(&b.index))
         });
 
+        if let Some(config) = &options.rerank {
+            results = Self::rerank_results(results, config, options.now);
+        }
+        if let Some(config) = &options.confidence {
+            let scored: Vec<ScoredResult> = results
+                .iter()
+                .map(|r| ScoredResult {
+                    index: r.index,
+                    score: r.score,
+                })
+                .collect();
+            let keep: HashSet<usize> = reject_low_confidence(&scored, config)
+                .into_iter()
+                .map(|r| r.index)
+                .collect();
+            results.retain(|r| keep.contains(&r.index));
+        }
+        results.truncate(k);
+
         // Only reinforce records that actually matched. When fewer than `k`
         // records are relevant, the rest of the list is zero-score filler;
         // boosting it would teach the store that arbitrary records are
@@ -164,11 +380,45 @@ impl HDF5Memory {
             .map(|r| r.index)
             .collect();
         self.apply_hebbian_boost(&hit_indices);
-        self.bm25 = Some(bm25);
 
         results
     }
 
+    /// Reorder by the re-ranker's combined score, which also becomes each
+    /// result's `score`.
+    fn rerank_results(
+        results: Vec<SearchResult>,
+        config: &ReRankConfig,
+        now: Option<f64>,
+    ) -> Vec<SearchResult> {
+        let now = now.unwrap_or_else(|| {
+            std::time::SystemTime::now()
+                .duration_since(std::time::UNIX_EPOCH)
+                .map(|d| d.as_secs_f64())
+                .unwrap_or(0.0)
+        });
+        let inputs: Vec<RerankInput> = results
+            .iter()
+            .map(|r| RerankInput {
+                index: r.index,
+                timestamp: r.timestamp,
+                source_channel: r.source_channel.clone(),
+                raw_activation: r.activation,
+                relevance: r.score,
+            })
+            .collect();
+        let mut by_index: std::collections::HashMap<usize, SearchResult> =
+            results.into_iter().map(|r| (r.index, r)).collect();
+        rerank(&inputs, config, now)
+            .into_iter()
+            .filter_map(|rr| {
+                let mut r = by_index.remove(&rr.index)?;
+                r.score = rr.combined_score;
+                Some(r)
+            })
+            .collect()
+    }
+
     /// Reinforce the records a query returned. The new weights are persisted by
     /// the next checkpoint (any write that flushes, `flush_wal`, or drop) — not
     /// by rewriting the whole store inside the query, which is what made

crates/clawhdf5-agent/src/session.rs

@@ -33,7 +33,7 @@ impl SessionCache {
         self.entries.is_empty()
     }
 
-    /// Add a new session with its summary.
+    /// Add a new session with its summary, timestamped now.
     pub fn add(
         &mut self,
         id: &str,
@@ -47,6 +47,21 @@ impl SessionCache {
             .unwrap_or_default()
             .as_secs_f64()
             * 1_000_000.0; // microseconds
+        self.add_at(id, start_idx, end_idx, channel, summary, ts);
+    }
+
+    /// Add a session with an explicit timestamp (Unix **microseconds**, the
+    /// unit [`SessionEntry::ts`] uses) — for importers carrying sessions over
+    /// from another store, whose original time should be kept.
+    pub fn add_at(
+        &mut self,
+        id: &str,
+        start_idx: usize,
+        end_idx: usize,
+        channel: &str,
+        summary: &str,
+        ts: f64,
+    ) {
         self.entries.push(SessionEntry {
             id: id.to_string(),
             start_idx: start_idx as u64,

crates/clawhdf5-agent/src/signing.rs

@@ -0,0 +1,419 @@
+//! Ed25519-signed checkpoints.
+//!
+//! When a signing key is set ([`crate::HDF5Memory::set_signing_key`]), every
+//! checkpoint writes a signed manifest of the store: a SHA-256 per memory
+//! record rolled into a Merkle root, plus hashes of the store's settings, its
+//! sessions and its knowledge graph. [`verify_store`] recomputes all of it from
+//! the file and checks the signature against a public key the caller trusts,
+//! so any change to the checkpointed file — a record's text or embedding, a
+//! setting, a session, a graph edge, made through this crate or any other HDF5
+//! tool — is detected, and the per-record hashes say which records changed.
+//!
+//! What it does not cover: saves still only in the WAL (made since the last
+//! checkpoint). [`VerifyReport::wal_entries_unsigned`] counts them.
+//!
+//! The hashes cover exactly what the file persists, in the form the loader
+//! returns it, so a store verifies after any number of reopen/checkpoint
+//! cycles. Derived data (L2 norms, the vector index) is not covered; it is
+//! recomputed from covered data.
+
+use ed25519_dalek::{Signature, Signer, Verifier};
+pub use ed25519_dalek::{SigningKey, VerifyingKey};
+use sha2::{Digest, Sha256};
+
+use crate::MemoryConfig;
+use crate::cache::MemoryCache;
+use crate::knowledge::KnowledgeCache;
+use crate::session::SessionCache;
+use crate::wal::WalMark;
+
+/// Version of the manifest encoding; part of what is signed.
+pub const MANIFEST_VERSION: i64 = 1;
+
+type Hash = [u8; 32];
+
+/// The hashes a signature covers.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct Manifest {
+    pub record_count: u64,
+    /// Merkle root over the per-record hashes.
+    pub records_root: Hash,
+    /// Settings persisted in `/meta`, plus the checkpoint's WAL mark.
+    pub settings: Hash,
+    pub sessions: Hash,
+    pub graph: Hash,
+}
+
+impl Manifest {
+    /// The exact bytes that are signed.
+    pub fn signed_bytes(&self) -> Vec<u8> {
+        let mut m = Vec::with_capacity(160);
+        m.extend_from_slice(b"clawhdf5-agent signed checkpoint\0");
+        m.extend_from_slice(&MANIFEST_VERSION.to_le_bytes());
+        m.extend_from_slice(&self.record_count.to_le_bytes());
+        m.extend_from_slice(&self.records_root);
+        m.extend_from_slice(&self.settings);
+        m.extend_from_slice(&self.sessions);
+        m.extend_from_slice(&self.graph);
+        m
+    }
+}
+
+/// A signature as stored in a checkpoint.
+#[derive(Debug, Clone)]
+pub struct StoredSignature {
+    pub manifest: Manifest,
+    pub record_hashes: Vec<Hash>,
+    pub public_key: [u8; 32],
+    pub signature: [u8; 64],
+}
+
+/// Build the manifest (and per-record hashes) for the state about to be
+/// checkpointed, and sign it.
+pub fn sign(
+    key: &SigningKey,
+    config: &MemoryConfig,
+    cache: &MemoryCache,
+    sessions: &SessionCache,
+    knowledge: &KnowledgeCache,
+    wal_applied: Option<WalMark>,
+) -> StoredSignature {
+    let (manifest, record_hashes) = manifest(config, cache, sessions, knowledge, wal_applied);
+    let signature = key.sign(&manifest.signed_bytes()).to_bytes();
+    StoredSignature {
+        manifest,
+        record_hashes,
+        public_key: key.verifying_key().to_bytes(),
+        signature,
+    }
+}
+
+/// Compute the manifest of a store's state.
+pub fn manifest(
+    config: &MemoryConfig,
+    cache: &MemoryCache,
+    sessions: &SessionCache,
+    knowledge: &KnowledgeCache,
+    wal_applied: Option<WalMark>,
+) -> (Manifest, Vec<Hash>) {
+    let record_hashes: Vec<Hash> = (0..cache.len()).map(|i| record_hash(cache, i)).collect();
+    let manifest = Manifest {
+        record_count: cache.len() as u64,
+        records_root: merkle_root(&record_hashes),
+        settings: settings_hash(config, wal_applied),
+        sessions: sessions_hash(sessions),
+        graph: graph_hash(knowledge),
+    };
+    (manifest, record_hashes)
+}
+
+// ---------------------------------------------------------------------------
+// Canonical encoding
+// ---------------------------------------------------------------------------
+
+/// A SHA-256 over length-prefixed fields, so no two different field lists
+/// hash the same bytes.
+struct Fields(Sha256);
+
+impl Fields {
+    fn new(domain: &str) -> Self {
+        let mut h = Sha256::new();
+        h.update((domain.len() as u64).to_le_bytes());
+        h.update(domain.as_bytes());
+        Self(h)
+    }
+    fn bytes(&mut self, b: &[u8]) -> &mut Self {
+        self.0.update((b.len() as u64).to_le_bytes());
+        self.0.update(b);
+        self
+    }
+    /// Strings as the loader returns them: stored null-padded, so a trailing
+    /// NUL cannot survive a round trip and must not be part of the hash.
+    fn str(&mut self, s: &str) -> &mut Self {
+        self.bytes(s.trim_end_matches('\0').as_bytes())
+    }
+    fn u64(&mut self, v: u64) -> &mut Self {
+        self.0.update(v.to_le_bytes());
+        self
+    }
+    fn f64(&mut self, v: f64) -> &mut Self {
+        self.0.update(v.to_bits().to_le_bytes());
+        self
+    }
+    fn f32(&mut self, v: f32) -> &mut Self {
+        self.0.update(v.to_bits().to_le_bytes());
+        self
+    }
+    fn finish(self) -> Hash {
+        self.0.finalize().into()
+    }
+}
+
+/// Everything persisted about record `i`, including its position. The
+/// embedding is hashed as the cache holds it — for a `float16` store that is
+/// the half-rounded value the file holds.
+fn record_hash(cache: &MemoryCache, i: usize) -> Hash {
+    let mut f = Fields::new("clawhdf5-agent/record");
+    f.u64(i as u64).str(&cache.chunks[i]);
+    let emb: Vec<u8> = cache.embeddings[i]
+        .iter()
+        .flat_map(|v| v.to_bits().to_le_bytes())
+        .collect();
+    f.bytes(&emb)
+        .str(&cache.source_channels[i])
+        .f64(cache.timestamps[i])
+        .str(&cache.session_ids[i])
+        .str(&cache.tags[i])
+        .u64(u64::from(cache.tombstones[i]))
+        .f32(cache.activation_weights[i]);
+    f.finish()
+}
+
+/// Binary Merkle tree: leaves are the record hashes; a parent hashes its two
+/// children with a node prefix; an odd node is carried up unchanged.
+fn merkle_root(leaves: &[Hash]) -> Hash {
+    if leaves.is_empty() {
+        return Fields::new("clawhdf5-agent/merkle-empty").finish();
+    }
+    let mut level: Vec<Hash> = leaves.to_vec();
+    while level.len() > 1 {
+        level = level
+            .chunks(2)
+            .map(|pair| match pair {
+                [l, r] => {
+                    let mut h = Sha256::new();
+                    h.update([1u8]);
+                    h.update(l);
+                    h.update(r);
+                    h.finalize().into()
+                }
+                [only] => *only,
+                _ => unreachable!(),
+            })
+            .collect();
+    }
+    level[0]
+}
+
+fn settings_hash(c: &MemoryConfig, wal_applied: Option<WalMark>) -> Hash {
+    let mut f = Fields::new("clawhdf5-agent/settings");
+    f.str(crate::schema::SCHEMA_VERSION)
+        .str(&c.created_at)
+        .str(&c.agent_id)
+        .str(&c.embedder)
+        .u64(c.embedding_dim as u64)
+        .u64(c.chunk_size as u64)
+        .u64(c.overlap as u64)
+        .u64(u64::from(c.float16))
+        .u64(u64::from(c.compression))
+        .u64(u64::from(c.compression_level))
+        .f32(c.compact_threshold)
+        .f32(c.hebbian_boost)
+        .f32(c.decay_factor)
+        .u64(u64::from(c.wal_enabled))
+        .u64(c.wal_max_entries as u64)
+        .u64(u64::from(c.quantized_index))
+        .u64(c.hnsw_m as u64)
+        .u64(c.hnsw_ef_construction as u64)
+        .u64(c.hnsw_ef_search as u64);
+    // An empty mark is not written to the file, so it must hash as none.
+    match wal_applied.filter(|m| m.len > 0) {
+        Some(m) => f.u64(1).u64(m.len).u64(u64::from(m.crc)),
+        None => f.u64(0),
+    };
+    f.finish()
+}
+
+fn sessions_hash(s: &SessionCache) -> Hash {
+    let mut f = Fields::new("clawhdf5-agent/sessions");
+    f.u64(s.entries.len() as u64);
+    for (i, e) in s.entries.iter().enumerate() {
+        f.str(&e.id)
+            .u64(e.start_idx)
+            .u64(e.end_idx)
+            .str(&e.channel)
+            .f64(e.ts)
+            .str(s.summaries.get(i).map(String::as_str).unwrap_or(""));
+    }
+    f.finish()
+}
+
+fn graph_hash(k: &KnowledgeCache) -> Hash {
+    let mut f = Fields::new("clawhdf5-agent/graph");
+    f.u64(k.entities.len() as u64);
+    for e in &k.entities {
+        f.u64(e.id)
+            .str(&e.name)
+            .str(&e.entity_type)
+            .u64(e.embedding_idx as u64);
+    }
+    f.u64(k.relations.len() as u64);
+    for r in &k.relations {
+        f.u64(r.src)
+            .u64(r.tgt)
+            .str(&r.relation)
+            .f32(r.weight)
+            .f64(r.ts);
+    }
+    f.u64(k.alias_strings.len() as u64);
+    for (s, id) in k.alias_strings.iter().zip(&k.alias_entity_ids) {
+        f.str(s).u64(*id as u64);
+    }
+    f.finish()
+}
+
+// ---------------------------------------------------------------------------
+// Verification
+// ---------------------------------------------------------------------------
+
+/// The outcome of [`verify_store`].
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct VerifyReport {
+    /// The checkpoint carries a signature.
+    pub signed: bool,
+    /// The signature was made by the key the caller trusts.
+    pub key_matches: bool,
+    /// The signature over the stored manifest is valid.
+    pub signature_valid: bool,
+    /// The file's current contents match the signed manifest.
+    pub records_match: bool,
+    pub settings_match: bool,
+    pub sessions_match: bool,
+    pub graph_match: bool,
+    /// Records whose contents differ from what was signed (by position),
+    /// when the stored per-record hashes are themselves authentic.
+    pub changed_records: Vec<usize>,
+    /// Records in the file versus in the signed manifest.
+    pub record_count: u64,
+    pub signed_record_count: u64,
+    /// The public key the checkpoint claims to be signed by.
+    pub public_key: Option<[u8; 32]>,
+    /// Saves in the WAL after the checkpoint: not covered by the signature.
+    pub wal_entries_unsigned: usize,
+}
+
+impl VerifyReport {
+    /// Signed by the trusted key, signature valid, and every part of the
+    /// file unchanged since it was signed.
+    pub fn is_valid(&self) -> bool {
+        self.signed
+            && self.key_matches
+            && self.signature_valid
+            && self.records_match
+            && self.settings_match
+            && self.sessions_match
+            && self.graph_match
+    }
+}
+
+/// Check a store file against the public key the caller trusts.
+///
+/// Reads the checkpoint (not the WAL), recomputes every hash from its
+/// contents and checks the signature. Never writes.
+pub fn verify_store(
+    path: &std::path::Path,
+    trusted: &VerifyingKey,
+) -> Result<VerifyReport, crate::MemoryError> {
+    let file = clawhdf5::File::open(path)
+        .map_err(|e| crate::MemoryError::Hdf5(format!("cannot open {}: {e}", path.display())))?;
+    let (config, cache, sessions, knowledge) = crate::schema::validate_and_load(&file)?;
+    let checkpoint = crate::schema::read_checkpoint_meta(&file);
+    let stored = crate::schema::read_signature(&file)?;
+    let wal_entries_unsigned = count_wal_entries_after(path, checkpoint.wal_applied);
+
+    let (current, current_hashes) = manifest(
+        &config,
+        &cache,
+        &sessions,
+        &knowledge,
+        checkpoint.wal_applied,
+    );
+
+    let Some(stored) = stored else {
+        return Ok(VerifyReport {
+            signed: false,
+            key_matches: false,
+            signature_valid: false,
+            records_match: false,
+            settings_match: false,
+            sessions_match: false,
+            graph_match: false,
+            changed_records: Vec::new(),
+            record_count: current.record_count,
+            signed_record_count: 0,
+            public_key: None,
+            wal_entries_unsigned,
+        });
+    };
+
+    let key_matches = stored.public_key == trusted.to_bytes();
+    let signature_valid = trusted
+        .verify(
+            &stored.manifest.signed_bytes(),
+            &Signature::from_bytes(&stored.signature),
+        )
+        .is_ok();
+    // The stored per-record hashes can localise a change only if they are
+    // the ones that were signed.
+    let hashes_authentic = signature_valid
+        && stored.record_hashes.len() as u64 == stored.manifest.record_count
+        && merkle_root(&stored.record_hashes) == stored.manifest.records_root;
+    let changed_records = if hashes_authentic {
+        let n = current_hashes.len().max(stored.record_hashes.len());
+        (0..n)
+            .filter(|&i| current_hashes.get(i) != stored.record_hashes.get(i))
+            .collect()
+    } else {
+        Vec::new()
+    };
+
+    Ok(VerifyReport {
+        signed: true,
+        key_matches,
+        signature_valid,
+        records_match: signature_valid
+            && current.record_count == stored.manifest.record_count
+            && current.records_root == stored.manifest.records_root,
+        settings_match: signature_valid && current.settings == stored.manifest.settings,
+        sessions_match: signature_valid && current.sessions == stored.manifest.sessions,
+        graph_match: signature_valid && current.graph == stored.manifest.graph,
+        changed_records,
+        record_count: current.record_count,
+        signed_record_count: stored.manifest.record_count,
+        public_key: Some(stored.public_key),
+        wal_entries_unsigned,
+    })
+}
+
+fn count_wal_entries_after(store: &std::path::Path, mark: Option<WalMark>) -> usize {
+    let wal = store.with_extension("h5.wal");
+    if !wal.exists() {
+        return 0;
+    }
+    crate::wal::WalFile::read_entries_for_migration(&wal, mark)
+        .map(|e| e.len())
+        .unwrap_or(0)
+}
+
+/// A new random signing key from the operating system's RNG.
+pub fn generate_key() -> SigningKey {
+    SigningKey::generate(&mut rand_core::OsRng)
+}
+
+/// Hex encoding for keys and signatures in attributes and the CLI.
+pub fn to_hex(bytes: &[u8]) -> String {
+    bytes.iter().map(|b| format!("{b:02x}")).collect()
+}
+
+/// Parse hex into exactly `N` bytes.
+pub fn from_hex<const N: usize>(s: &str) -> Option<[u8; N]> {
+    let s = s.trim();
+    if s.len() != 2 * N {
+        return None;
+    }
+    let mut out = [0u8; N];
+    for (i, byte) in out.iter_mut().enumerate() {
+        *byte = u8::from_str_radix(&s[2 * i..2 * i + 2], 16).ok()?;
+    }
+    Some(out)
+}

crates/clawhdf5-agent/src/storage.rs

@@ -36,7 +36,7 @@ pub fn write_to_disk_with_mark(
 ) -> Result<(), MemoryError> {
     let meta = schema::CheckpointMeta {
         wal_applied,
-        ann_generation: None,
+        ..schema::CheckpointMeta::default()
     };
     write_to_disk_with_meta(path, config, cache, sessions, knowledge, &meta)
 }
@@ -50,7 +50,21 @@ pub fn write_to_disk_with_meta(
     knowledge: &KnowledgeCache,
     checkpoint: &schema::CheckpointMeta,
 ) -> Result<(), MemoryError> {
-    let bytes = schema::build_hdf5_file_with_meta(config, cache, sessions, knowledge, checkpoint)?;
+    write_to_disk_signed(path, config, cache, sessions, knowledge, checkpoint, None)
+}
+
+/// [`write_to_disk_with_meta`] with a signed manifest of the contents.
+pub fn write_to_disk_signed(
+    path: &Path,
+    config: &MemoryConfig,
+    cache: &MemoryCache,
+    sessions: &SessionCache,
+    knowledge: &KnowledgeCache,
+    checkpoint: &schema::CheckpointMeta,
+    signature: Option<&crate::signing::StoredSignature>,
+) -> Result<(), MemoryError> {
+    let bytes =
+        schema::build_hdf5_file_signed(config, cache, sessions, knowledge, checkpoint, signature)?;
 
     if bytes.is_empty() {
         return Err(MemoryError::Hdf5("build_hdf5_file produced 0 bytes".into()));

crates/clawhdf5-agent/tests/float16_store.rs

@@ -0,0 +1,260 @@
+//! `MemoryConfig::float16`: embeddings stored as IEEE half precision.
+//!
+//! The setting used to be recorded in `/meta` and otherwise ignored — the
+//! embeddings dataset was always `f32`. These tests pin what it now does: the
+//! dataset is `float16`, the in-memory cache holds exactly the values the file
+//! holds (so search results survive a reopen bit for bit), and a value half
+//! precision cannot represent is refused rather than stored as infinity.
+
+use std::path::{Path, PathBuf};
+
+use clawhdf5_agent::{AgentMemory, HDF5Memory, MemoryConfig, MemoryEntry, MemoryError};
+use clawhdf5_format::float16::round_to_f16;
+use tempfile::TempDir;
+
+const DIM: usize = 64;
+
+/// Deterministic, embedding-like unit vectors.
+fn embedding(seed: u64) -> Vec<f32> {
+    let mut x = seed.wrapping_mul(0x9E37_79B9_7F4A_7C15) | 1;
+    let v: Vec<f32> = (0..DIM)
+        .map(|_| {
+            x ^= x << 13;
+            x ^= x >> 7;
+            x ^= x << 17;
+            (x >> 40) as f32 / (1u64 << 24) as f32 - 0.5
+        })
+        .collect();
+    let norm = v.iter().map(|a| a * a).sum::<f32>().sqrt();
+    v.iter().map(|a| a / norm).collect()
+}
+
+fn entry(i: u64) -> MemoryEntry {
+    MemoryEntry {
+        chunk: format!("memory number {i} about topic {}", i % 7),
+        embedding: embedding(i),
+        source_channel: "test".into(),
+        timestamp: i as f64,
+        session_id: "s".into(),
+        tags: format!("t{i}"),
+    }
+}
+
+fn config(dir: &TempDir, name: &str, float16: bool) -> MemoryConfig {
+    let mut c = MemoryConfig::new(dir.path().join(name), "agent", DIM);
+    c.float16 = float16;
+    c
+}
+
+fn embeddings_dtype_and_values(path: &Path) -> (String, Vec<f32>) {
+    let file = clawhdf5::File::open(path).unwrap();
+    let ds = file.dataset("memory/embeddings").unwrap();
+    (format!("{:?}", ds.dtype().unwrap()), ds.read_f32().unwrap())
+}
+
+fn search_bits(m: &mut HDF5Memory, q: u64) -> Vec<(usize, u32)> {
+    m.hybrid_search(&embedding(q), "memory topic 3", 0.4, 0.6, 10)
+        .iter()
+        .map(|r| (r.index, r.score.to_bits()))
+        .collect()
+}
+
+#[test]
+fn float16_store_writes_half_precision_and_reopens_identically() {
+    let dir = TempDir::new().unwrap();
+    // Two identical stores. Search is not read-only (it boosts the Hebbian
+    // activation of what it returns, and checkpoints persist that), so each
+    // is queried exactly once: one live, one after a checkpoint and reopen.
+    let live_cfg = config(&dir, "live.h5", true);
+    let cfg = config(&dir, "f16.h5", true);
+    let path: PathBuf = cfg.path.clone();
+
+    let mut live = HDF5Memory::create(live_cfg).unwrap();
+    live.save_batch((0..200).map(entry).collect()).unwrap();
+    let mut m = HDF5Memory::create(cfg).unwrap();
+    m.save_batch((0..200).map(entry).collect()).unwrap();
+    drop(m);
+
+    // On disk: a genuine float16 dataset holding the rounded inputs.
+    let (dtype, values) = embeddings_dtype_and_values(&path);
+    assert_eq!(dtype, "Other(\"float16\")");
+    let expected: Vec<u32> = (0..200)
+        .flat_map(|i| embedding(i).into_iter().map(|v| round_to_f16(v).to_bits()))
+        .collect();
+    let got: Vec<u32> = values.iter().map(|v| v.to_bits()).collect();
+    assert_eq!(got, expected);
+
+    // Reopened, the store answers exactly as the live one does: the cache
+    // held the half-rounded values before the checkpoint.
+    let mut reopened = HDF5Memory::open(&path).unwrap();
+    for q in 0..5 {
+        assert_eq!(
+            search_bits(&mut live, 1000 + q),
+            search_bits(&mut reopened, 1000 + q),
+            "query {q}"
+        );
+    }
+}
+
+#[test]
+fn float16_halves_the_embeddings_on_disk() {
+    let dir = TempDir::new().unwrap();
+    let mut sizes = Vec::new();
+    for float16 in [false, true] {
+        let cfg = config(&dir, &format!("s{float16}.h5"), float16);
+        let path = cfg.path.clone();
+        let mut m = HDF5Memory::create(cfg).unwrap();
+        m.save_batch((0..2000).map(entry).collect()).unwrap();
+        drop(m);
+        sizes.push(std::fs::metadata(&path).unwrap().len());
+    }
+    let embedding_bytes_f32 = (2000 * DIM * 4) as u64;
+    let saved = sizes[0] - sizes[1];
+    // Half of the f32 embeddings, give or take metadata and alignment.
+    assert!(
+        saved.abs_diff(embedding_bytes_f32 / 2) < 16 * 1024,
+        "f32 {} B, f16 {} B, saved {saved} B, expected ~{} B",
+        sizes[0],
+        sizes[1],
+        embedding_bytes_f32 / 2
+    );
+}
+
+#[test]
+fn f32_store_is_unchanged() {
+    let dir = TempDir::new().unwrap();
+    let cfg = config(&dir, "f32.h5", false);
+    let path = cfg.path.clone();
+    let mut m = HDF5Memory::create(cfg).unwrap();
+    m.save_batch((0..50).map(entry).collect()).unwrap();
+    drop(m);
+    let (dtype, values) = embeddings_dtype_and_values(&path);
+    assert_eq!(dtype, "F32");
+    let expected: Vec<f32> = (0..50).flat_map(embedding).collect();
+    assert_eq!(values, expected);
+}
+
+#[test]
+fn out_of_range_values_are_refused_not_stored_as_infinity() {
+    let dir = TempDir::new().unwrap();
+    let mut cfg = config(&dir, "range.h5", true);
+    cfg.wal_enabled = true;
+    let path = cfg.path.clone();
+    let mut m = HDF5Memory::create(cfg).unwrap();
+    m.save(entry(1)).unwrap();
+
+    let mut bad = entry(2);
+    bad.embedding[5] = 70_000.0;
+    match m.save(bad.clone()) {
+        Err(MemoryError::InvalidEntry(msg)) => assert!(msg.contains("embedding[5]"), "{msg}"),
+        other => panic!("expected InvalidEntry, got {other:?}"),
+    }
+    assert!(matches!(
+        m.save_or_update(bad.clone()),
+        Err(MemoryError::InvalidEntry(_))
+    ));
+    // A batch is all or nothing.
+    assert!(matches!(
+        m.save_batch(vec![entry(3), bad.clone(), entry(4)]),
+        Err(MemoryError::InvalidEntry(_))
+    ));
+    assert_eq!(m.count(), 1);
+
+    // The largest finite half, and values that round down to it, are fine.
+    let mut edge = entry(5);
+    edge.embedding[0] = 65504.0;
+    edge.embedding[1] = -65519.0;
+    m.save(edge).unwrap();
+    assert_eq!(m.count(), 2);
+    drop(m);
+
+    // Nothing rejected reached the WAL or the file.
+    let m = HDF5Memory::open(&path).unwrap();
+    assert_eq!(m.count(), 2);
+
+    // An f32 store takes the same value as it always did.
+    let mut m32 = HDF5Memory::create(config(&dir, "range32.h5", false)).unwrap();
+    m32.save(bad).unwrap();
+}
+
+#[test]
+fn wal_replay_rounds_like_a_live_save() {
+    let dir = TempDir::new().unwrap();
+    let mut cfg = config(&dir, "wal.h5", true);
+    cfg.wal_enabled = true;
+    cfg.wal_max_entries = 10_000; // keep everything in the WAL
+    let path = cfg.path.clone();
+    let mut m = HDF5Memory::create(cfg).unwrap();
+    for i in 0..30 {
+        m.save(entry(i)).unwrap();
+    }
+    let live = search_bits(&mut m, 77);
+
+    // Crash image: the .h5 is still the empty checkpoint; everything is in
+    // the WAL, which holds the caller's f32 values.
+    let crash = TempDir::new().unwrap();
+    let image = crash.path().join("image.h5");
+    std::fs::copy(&path, &image).unwrap();
+    std::fs::copy(
+        path.with_extension("h5.wal"),
+        image.with_extension("h5.wal"),
+    )
+    .unwrap();
+    drop(m);
+
+    let mut recovered = HDF5Memory::open(&image).unwrap();
+    assert_eq!(recovered.count(), 30);
+    assert_eq!(search_bits(&mut recovered, 77), live);
+}
+
+#[test]
+fn new_stores_default_to_float16() {
+    let dir = TempDir::new().unwrap();
+    let path = dir.path().join("default.h5");
+    let mut m = HDF5Memory::create(MemoryConfig::new(path.clone(), "agent", DIM)).unwrap();
+    assert!(m.config().float16);
+    m.save_batch((0..10).map(entry).collect()).unwrap();
+    drop(m);
+    assert_eq!(embeddings_dtype_and_values(&path).0, "Other(\"float16\")");
+    assert!(HDF5Memory::open(&path).unwrap().config().float16);
+}
+
+#[test]
+fn an_existing_f32_store_stays_f32() {
+    // Written by the v2.5.0 CLI, with `float16 = 0` in /meta (every agent
+    // store has recorded it). Flipping the default for new stores must not
+    // reach back and round an existing store's embeddings.
+    let dir = TempDir::new().unwrap();
+    let path = dir.path().join("legacy.h5");
+    std::fs::copy(
+        concat!(
+            env!("CARGO_MANIFEST_DIR"),
+            "/tests/fixtures/store_v2_5_0.h5"
+        ),
+        &path,
+    )
+    .unwrap();
+    let before = embeddings_dtype_and_values(&path);
+    assert_eq!(before.0, "F32");
+
+    let mut m = HDF5Memory::open(&path).unwrap();
+    assert!(!m.config().float16, "an old store must reopen as f32");
+    let dim = m.config().embedding_dim;
+    let odd: Vec<f32> = (0..dim).map(|i| 0.1 + i as f32 * 1e-4).collect();
+    m.save_batch(vec![MemoryEntry {
+        chunk: "added after the upgrade".into(),
+        embedding: odd.clone(),
+        source_channel: "test".into(),
+        timestamp: 1.0,
+        session_id: "s".into(),
+        tags: String::new(),
+    }])
+    .unwrap();
+    drop(m);
+
+    // Checkpointed: still f32, the old rows untouched and the new one exact.
+    let (dtype, values) = embeddings_dtype_and_values(&path);
+    assert_eq!(dtype, "F32");
+    assert_eq!(&values[..before.1.len()], before.1.as_slice());
+    assert_eq!(&values[before.1.len()..], odd.as_slice());
+}

crates/clawhdf5-agent/tests/h5py_interop.rs

@@ -0,0 +1,155 @@
+//! An agent store is a standard HDF5 file: h5py can open it and read every
+//! dataset.
+//!
+//! It could not: the float datatype's sign-bit position was hard-coded for
+//! f64, so every f32 dataset (embeddings, norms, activation weights) made
+//! libhdf5 refuse the file with "sign bit position out of bounds".
+
+use std::process::Command;
+
+use clawhdf5_agent::{AgentMemory, HDF5Memory, MemoryConfig, MemoryEntry};
+
+fn python() -> String {
+    std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
+}
+
+fn h5py_available() -> bool {
+    Command::new(python())
+        .args(["-c", "import h5py"])
+        .output()
+        .map(|o| o.status.success())
+        .unwrap_or(false)
+}
+
+#[test]
+fn h5py_reads_every_dataset_of_an_agent_store() {
+    if !h5py_available() {
+        assert!(
+            std::env::var("CLAWHDF5_REQUIRE_INTEROP").as_deref() != Ok("1"),
+            "CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available"
+        );
+        eprintln!("SKIP: python3 with h5py not available");
+        return;
+    }
+    let dir = tempfile::tempdir().unwrap();
+    for float16 in [false, true] {
+        let path = dir.path().join(format!("store_{float16}.h5"));
+        let mut cfg = MemoryConfig::new(path.clone(), "agent", 8);
+        cfg.float16 = float16;
+        let mut m = HDF5Memory::create(cfg).unwrap();
+        // save_batch checkpoints, so the records are in the .h5, not the WAL.
+        m.save_batch(
+            (0..20)
+                .map(|i| MemoryEntry {
+                    chunk: format!("memory {i}"),
+                    embedding: (0..8).map(|j| ((i * 8 + j) as f32).sin()).collect(),
+                    source_channel: "test".into(),
+                    timestamp: i as f64,
+                    session_id: "s".into(),
+                    tags: String::new(),
+                })
+                .collect(),
+        )
+        .unwrap();
+        drop(m);
+
+        // Exact expected values, as bits: numpy's sin need not match Rust's
+        // to the last place.
+        let bits = (0..160)
+            .map(|k| (k as f32).sin().to_bits().to_string())
+            .collect::<Vec<_>>()
+            .join(",");
+        let script = format!(
+            r#"
+import h5py, numpy as np
+want = np.float16 if {py_bool} else np.float32
+with h5py.File("{path}", "r") as f:
+    names = []
+    f.visititems(lambda n, o: names.append(n) if isinstance(o, h5py.Dataset) else None)
+    for n in names:
+        f[n][()]  # every dataset must decode
+    e = f["memory/embeddings"]
+    assert e.dtype == want, e.dtype
+    assert e.shape == (20, 8), e.shape
+    ref = np.array([{bits}], dtype=np.uint32).view(np.float32).astype(want).reshape(20, 8)
+    assert (e[()] == ref).all()
+    assert f["memory/norms"].dtype == np.float32
+print(len(names))
+"#,
+            py_bool = if float16 { "True" } else { "False" },
+            path = path.display()
+        );
+        let out = Command::new(python())
+            .args(["-c", &script])
+            .output()
+            .unwrap();
+        assert!(
+            out.status.success(),
+            "float16={float16}: {}",
+            String::from_utf8_lossy(&out.stderr)
+        );
+        let n: usize = String::from_utf8_lossy(&out.stdout).trim().parse().unwrap();
+        assert!(n >= 10, "only {n} datasets");
+    }
+}
+
+#[test]
+fn an_edit_made_with_h5py_breaks_the_signature_and_names_the_record() {
+    if !h5py_available() {
+        assert!(
+            std::env::var("CLAWHDF5_REQUIRE_INTEROP").as_deref() != Ok("1"),
+            "CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available"
+        );
+        eprintln!("SKIP: python3 with h5py not available");
+        return;
+    }
+    use clawhdf5_agent::signing::SigningKey;
+    let dir = tempfile::tempdir().unwrap();
+    let path = dir.path().join("signed.h5");
+    let key = SigningKey::from_bytes(&[42; 32]);
+    let mut m = HDF5Memory::create(MemoryConfig::new(path.clone(), "agent", 8)).unwrap();
+    m.set_signing_key(key.clone());
+    m.save_batch(
+        (0..10)
+            .map(|i| MemoryEntry {
+                chunk: format!("memory {i}"),
+                embedding: (0..8).map(|j| ((i * 8 + j) as f32).cos()).collect(),
+                source_channel: "test".into(),
+                timestamp: i as f64,
+                session_id: "s".into(),
+                tags: String::new(),
+            })
+            .collect(),
+    )
+    .unwrap();
+    drop(m);
+    assert!(
+        HDF5Memory::verify(&path, &key.verifying_key())
+            .unwrap()
+            .is_valid()
+    );
+
+    // Someone edits one timestamp in place with h5py.
+    let script = format!(
+        r#"
+import h5py
+with h5py.File("{}", "r+") as f:
+    ts = f["memory/timestamps"]
+    ts[3] = 12345.0
+"#,
+        path.display()
+    );
+    let out = Command::new(python())
+        .args(["-c", &script])
+        .output()
+        .unwrap();
+    assert!(
+        out.status.success(),
+        "{}",
+        String::from_utf8_lossy(&out.stderr)
+    );
+
+    let r = HDF5Memory::verify(&path, &key.verifying_key()).unwrap();
+    assert!(r.signature_valid && !r.is_valid(), "{r:?}");
+    assert_eq!(r.changed_records, vec![3]);
+}

crates/clawhdf5-agent/tests/hnsw_integration.rs

@@ -284,3 +284,63 @@ fn degenerate_hnsw_parameters_do_not_panic() {
     let results = mem.hybrid_search(&vectors[7], "", 1.0, 0.0, 5);
     assert_eq!(results[0].index, 7, "exact match should still rank first");
 }
+
+#[test]
+fn new_stores_default_to_the_quantized_index() {
+    // int8 is the default because it is smaller and, with an exact re-score,
+    // faster at equal recall on every platform measured (see BENCHMARKS.md).
+    let dir = TempDir::new().unwrap();
+    let config = MemoryConfig::new(dir.path().join("mem.h5"), "agent", 8);
+    assert!(config.quantized_index);
+
+    let path = config.path.clone();
+    let mut mem = HDF5Memory::create(config).unwrap();
+    let mut seed = 3;
+    let vectors: Vec<Vec<f32>> = (0..40).map(|_| make_vector(&mut seed, 8)).collect();
+    for (i, v) in vectors.iter().enumerate() {
+        mem.save(entry(&format!("c{i}"), v.clone(), "t")).unwrap();
+    }
+    assert_eq!(
+        mem.hybrid_search(&vectors[11], "", 1.0, 0.0, 1)[0].index,
+        11
+    );
+    mem.flush_wal().unwrap();
+    drop(mem);
+    assert!(HDF5Memory::open(&path).unwrap().config().quantized_index);
+}
+
+#[test]
+fn a_store_written_before_the_setting_existed_stays_f32() {
+    // `store_v2_5_0.h5` was written by the v2.5.0 CLI, before
+    // `quantized_index` or the HNSW parameters were persisted, so it carries
+    // none of them. Flipping the default for new stores must not reach back
+    // and change how an existing store's index is held.
+    let dir = TempDir::new().unwrap();
+    let path = dir.path().join("legacy.h5");
+    std::fs::copy(
+        concat!(
+            env!("CARGO_MANIFEST_DIR"),
+            "/tests/fixtures/store_v2_5_0.h5"
+        ),
+        &path,
+    )
+    .unwrap();
+
+    let bytes = std::fs::read(&path).unwrap();
+    assert!(
+        !bytes.windows(15).any(|w| w == b"quantized_index"),
+        "the fixture must predate the setting, or it tests nothing"
+    );
+
+    let mut mem = HDF5Memory::open(&path).unwrap();
+    assert!(
+        !mem.config().quantized_index,
+        "an old store must reopen with an f32 index"
+    );
+    assert_eq!(mem.config().hnsw_m, 16);
+    assert_eq!(mem.config().hnsw_ef_construction, 64);
+    assert_eq!(mem.count(), 6);
+    // And it still searches: entry 3's own embedding finds it first.
+    let hit = mem.hybrid_search(&[3.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], "", 1.0, 0.0, 1);
+    assert_eq!(hit[0].index, 3);
+}

crates/clawhdf5-agent/tests/search_options.rs

@@ -0,0 +1,344 @@
+//! `HDF5Memory::search` with `SearchOptions`: source filtering, re-ranking and
+//! confidence rejection in the store's own search path.
+
+use std::collections::HashSet;
+
+use clawhdf5_agent::confidence::ConfidenceConfig;
+use clawhdf5_agent::reranker::ReRankConfig;
+use clawhdf5_agent::{AgentMemory, HDF5Memory, MemoryConfig, MemoryEntry, SearchOptions, hybrid};
+use tempfile::TempDir;
+
+const DIM: usize = 32;
+const N: usize = 3000;
+const CLUSTERS: usize = 20;
+
+struct Rng(u64);
+impl Rng {
+    fn next(&mut self) -> u64 {
+        self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15);
+        let mut z = self.0;
+        z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
+        z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
+        z ^ (z >> 31)
+    }
+    fn unit(&mut self) -> f32 {
+        (self.next() >> 40) as f32 / (1u64 << 24) as f32 - 0.5
+    }
+}
+
+fn normalize(v: &mut [f32]) {
+    let n = v.iter().map(|x| x * x).sum::<f32>().sqrt();
+    v.iter_mut().for_each(|x| *x /= n);
+}
+
+struct Data {
+    vectors: Vec<Vec<f32>>,
+    cluster: Vec<usize>,
+    centres: Vec<Vec<f32>>,
+}
+
+fn data() -> Data {
+    let mut rng = Rng(42);
+    let centres: Vec<Vec<f32>> = (0..CLUSTERS)
+        .map(|_| {
+            let mut c: Vec<f32> = (0..DIM).map(|_| rng.unit()).collect();
+            normalize(&mut c);
+            c
+        })
+        .collect();
+    let mut vectors = Vec::new();
+    let mut cluster = Vec::new();
+    for i in 0..N {
+        let c = i % CLUSTERS;
+        let mut v: Vec<f32> = centres[c].iter().map(|x| x + rng.unit() * 0.3).collect();
+        normalize(&mut v);
+        vectors.push(v);
+        cluster.push(c);
+    }
+    Data {
+        vectors,
+        cluster,
+        centres,
+    }
+}
+
+/// Channel of record `i` for a filter keeping `percent`% of the store at
+/// random (independent of the vectors).
+fn random_channel(i: usize, rng_seed: u64, percent: u64) -> String {
+    let mut r = Rng(rng_seed ^ (i as u64 * 7919));
+    if r.next() % 100 < percent {
+        "keep".into()
+    } else {
+        "other".into()
+    }
+}
+
+fn build(data: &Data, channel: impl Fn(usize) -> String) -> (TempDir, HDF5Memory) {
+    let dir = TempDir::new().unwrap();
+    let mut cfg = MemoryConfig::new(dir.path().join("s.h5"), "agent", DIM);
+    cfg.hebbian_boost = 0.0; // every query sees the same store
+    let mut m = HDF5Memory::create(cfg).unwrap();
+    let entries = data
+        .vectors
+        .iter()
+        .enumerate()
+        .map(|(i, v)| MemoryEntry {
+            chunk: format!("record {i} cluster {}", data.cluster[i]),
+            embedding: v.clone(),
+            source_channel: channel(i),
+            timestamp: i as f64,
+            session_id: "s".into(),
+            tags: format!("t{i}"),
+        })
+        .collect();
+    m.save_batch(entries).unwrap();
+    (dir, m)
+}
+
+/// Exact top-k by cosine among the records `allowed` keeps.
+fn exact_top(data: &Data, q: &[f32], k: usize, allowed: impl Fn(usize) -> bool) -> Vec<usize> {
+    let mut s: Vec<(usize, f32)> = (0..N)
+        .filter(|&i| allowed(i))
+        .map(|i| (i, data.vectors[i].iter().zip(q).map(|(a, b)| a * b).sum()))
+        .collect();
+    s.sort_by(|a, b| b.1.total_cmp(&a.1).then(a.0.cmp(&b.0)));
+    s.into_iter().take(k).map(|(i, _)| i).collect()
+}
+
+fn query(data: &Data, i: usize) -> Vec<f32> {
+    let mut rng = Rng(1000 + i as u64);
+    let mut q: Vec<f32> = data.centres[i % CLUSTERS]
+        .iter()
+        .map(|x| x + rng.unit() * 0.3)
+        .collect();
+    normalize(&mut q);
+    q
+}
+
+fn vector_only(k: usize) -> SearchOptions {
+    SearchOptions::new(k).with_fusion(hybrid::Fusion::Weighted {
+        vector: 1.0,
+        keyword: 0.0,
+    })
+}
+
+#[test]
+fn source_filter_returns_only_allowed_records_and_a_full_page() {
+    let d = data();
+    // At N = 3000 and k = 10 the index serves a filter only when that is
+    // cheaper than scanning the allowed records: pool = 80 * N / allowed
+    // candidates at ~M = 16 distances each, against `allowed` distances. So
+    // 90% goes through the index, 50% and 1% to the exact scan.
+    for percent in [90, 50, 1] {
+        let (_dir, mut m) = build(&d, |i| random_channel(i, 5, percent));
+        let allowed = |i: usize| random_channel(i, 5, percent) == "keep";
+        let mut hits = 0;
+        for qi in 0..40 {
+            let q = query(&d, qi);
+            let got = m.search(&q, "", &vector_only(10).with_sources(["keep"]));
+            assert_eq!(got.len(), 10, "{percent}%: short page");
+            assert!(got.iter().all(|r| r.source_channel == "keep"));
+            let want: HashSet<usize> = exact_top(&d, &q, 10, allowed).into_iter().collect();
+            hits += got.iter().filter(|r| want.contains(&r.index)).count();
+        }
+        let recall = hits as f64 / 400.0;
+        let floor = if percent == 90 { 0.95 } else { 1.0 };
+        assert!(recall >= floor, "{percent}%: recall@10 {recall}");
+    }
+}
+
+#[test]
+fn filter_away_from_the_query_falls_back_to_an_exact_scan() {
+    // Channel = cluster, and the filter keeps two clusters (10% of the
+    // store) that are not the query's: the index's neighbourhood of the
+    // query holds none of them. The search must still return the exact
+    // top 10 among the allowed records, not a short or empty page.
+    let d = data();
+    let (_dir, mut m) = build(&d, |i| format!("c{}", d.cluster[i]));
+    for qi in 0..20 {
+        let q = query(&d, qi);
+        let a = format!("c{}", (qi + 7) % CLUSTERS);
+        let b = format!("c{}", (qi + 13) % CLUSTERS);
+        let got: Vec<usize> = m
+            .search(
+                &q,
+                "",
+                &vector_only(10).with_sources([a.clone(), b.clone()]),
+            )
+            .iter()
+            .map(|r| r.index)
+            .collect();
+        let want = exact_top(&d, &q, 10, |i| {
+            let c = format!("c{}", d.cluster[i]);
+            c == a || c == b
+        });
+        assert_eq!(got, want, "query {qi}");
+    }
+}
+
+#[test]
+fn filter_edge_cases() {
+    let d = data();
+    let (_dir, mut m) = build(&d, |i| random_channel(i, 9, 50));
+    let q = query(&d, 0);
+    assert!(
+        m.search(
+            &q,
+            "cluster",
+            &SearchOptions::new(10).with_sources(Vec::<String>::new())
+        )
+        .is_empty()
+    );
+    assert!(
+        m.search(
+            &q,
+            "cluster",
+            &SearchOptions::new(10).with_sources(["nope"])
+        )
+        .is_empty()
+    );
+    // Keyword matches from other channels are filtered too.
+    let got = m.search(
+        &q,
+        "record cluster",
+        &SearchOptions::new(50).with_sources(["keep"]),
+    );
+    assert_eq!(got.len(), 50);
+    assert!(got.iter().all(|r| r.source_channel == "keep"));
+    // Deleted records never come back, filtered or not.
+    let first = got[0].index;
+    m.delete(first).unwrap();
+    let again = m.search(
+        &q,
+        "record cluster",
+        &SearchOptions::new(50).with_sources(["keep"]),
+    );
+    assert!(again.iter().all(|r| r.index != first));
+}
+
+#[test]
+fn plain_options_equal_hybrid_search_with() {
+    // Two identical stores, so neither query sees the other's boosts.
+    let d = data();
+    let (_a, mut a) = build(&d, |i| random_channel(i, 3, 50));
+    let (_b, mut b) = build(&d, |i| random_channel(i, 3, 50));
+    for qi in 0..10 {
+        let q = query(&d, qi);
+        let x: Vec<(usize, u32)> = a
+            .search(&q, "record cluster 3", &SearchOptions::new(10))
+            .iter()
+            .map(|r| (r.index, r.score.to_bits()))
+            .collect();
+        let y: Vec<(usize, u32)> = b
+            .hybrid_search_with(&q, "record cluster 3", hybrid::DEFAULT_FUSION, 10)
+            .iter()
+            .map(|r| (r.index, r.score.to_bits()))
+            .collect();
+        assert_eq!(x, y);
+    }
+}
+
+fn small_store(entries: &[(&str, &str, f64)]) -> (TempDir, HDF5Memory) {
+    let dir = TempDir::new().unwrap();
+    let mut m = HDF5Memory::create(MemoryConfig::new(dir.path().join("r.h5"), "a", 4)).unwrap();
+    m.save_batch(
+        entries
+            .iter()
+            .map(|(chunk, channel, ts)| MemoryEntry {
+                chunk: chunk.to_string(),
+                embedding: vec![1.0, 0.0, 0.0, 0.0],
+                source_channel: channel.to_string(),
+                timestamp: *ts,
+                session_id: "s".into(),
+                tags: String::new(),
+            })
+            .collect(),
+    )
+    .unwrap();
+    (dir, m)
+}
+
+#[test]
+fn rerank_breaks_relevance_ties_by_recency() {
+    // Identical text and vectors, so retrieval ties; re-ranking must put the
+    // newer record first and report the combined score.
+    let now = 1_000_000.0;
+    let (_d, mut m) = small_store(&[
+        ("user prefers dark mode", "chat", now - 30.0 * 86_400.0),
+        ("user prefers dark mode", "chat", now - 60.0),
+    ]);
+    let q = [1.0, 0.0, 0.0, 0.0];
+    let plain = m.search(&q, "dark mode", &SearchOptions::new(2));
+    assert_eq!(plain[0].index, 0, "ties break by index without re-ranking");
+    let reranked = m.search(
+        &q,
+        "dark mode",
+        &SearchOptions::new(2)
+            .with_rerank(ReRankConfig::default())
+            .at_time(now),
+    );
+    assert_eq!(reranked[0].index, 1);
+    assert!(reranked[0].score > reranked[1].score);
+    assert_ne!(reranked[0].score.to_bits(), plain[0].score.to_bits());
+}
+
+#[test]
+fn confidence_rejects_when_nothing_is_good_enough() {
+    let (_d, mut m) = small_store(&[("alpha", "chat", 0.0), ("beta", "chat", 0.0)]);
+    let q = [1.0, 0.0, 0.0, 0.0];
+    let strict = ConfidenceConfig {
+        min_score: 10.0,
+        ..ConfidenceConfig::default()
+    };
+    assert!(
+        m.search(&q, "alpha", &SearchOptions::new(2).with_confidence(strict))
+            .is_empty()
+    );
+    let lenient = ConfidenceConfig {
+        min_score: 0.0,
+        min_gap: f32::INFINITY,
+        max_results: 1,
+    };
+    assert_eq!(
+        m.search(&q, "alpha", &SearchOptions::new(2).with_confidence(lenient))
+            .len(),
+        1
+    );
+}
+
+#[test]
+fn only_returned_results_are_reinforced() {
+    // With re-ranking, a pool of max(3k, 10) candidates is retrieved; only
+    // the k returned should gain activation.
+    let d = data();
+    let dir = TempDir::new().unwrap();
+    let path = dir.path().join("h.h5");
+    let mut m = HDF5Memory::create(MemoryConfig::new(path, "a", DIM)).unwrap();
+    m.save_batch(
+        (0..200)
+            .map(|i| MemoryEntry {
+                chunk: format!("record {i}"),
+                embedding: d.vectors[i].clone(),
+                source_channel: "chat".into(),
+                timestamp: i as f64,
+                session_id: "s".into(),
+                tags: String::new(),
+            })
+            .collect(),
+    )
+    .unwrap();
+    let q = query(&d, 0);
+    let got = m.search(
+        &q,
+        "record",
+        &SearchOptions::new(3).with_rerank(ReRankConfig::default()),
+    );
+    assert_eq!(got.len(), 3);
+    let returned: HashSet<usize> = got.iter().map(|r| r.index).collect();
+    // A second plain search reports each record's current activation.
+    let all = m.search(&q, "record", &SearchOptions::new(200));
+    for r in &all {
+        let boosted = r.activation > 1.0;
+        assert_eq!(boosted, returned.contains(&r.index), "record {}", r.index);
+    }
+}

crates/clawhdf5-agent/tests/signed_store.rs

@@ -0,0 +1,330 @@
+//! Ed25519-signed checkpoints: `HDF5Memory::set_signing_key` and
+//! `HDF5Memory::verify`.
+
+use std::path::Path;
+
+use clawhdf5_agent::signing::{SigningKey, VerifyReport, VerifyingKey};
+use clawhdf5_agent::storage;
+use clawhdf5_agent::{AgentMemory, HDF5Memory, MemoryConfig, MemoryEntry, MemoryError, schema};
+use tempfile::TempDir;
+
+const DIM: usize = 16;
+
+fn key(seed: u8) -> SigningKey {
+    SigningKey::from_bytes(&[seed; 32])
+}
+
+fn entry(i: usize, chunk: &str) -> MemoryEntry {
+    MemoryEntry {
+        chunk: chunk.to_string(),
+        embedding: (0..DIM)
+            .map(|j| ((i * DIM + j) as f32 * 0.37).sin())
+            .collect(),
+        source_channel: "chat".into(),
+        timestamp: 1_700_000_000.0 + i as f64,
+        session_id: format!("s{}", i % 3),
+        tags: format!("t{i}"),
+    }
+}
+
+/// Awkward strings on purpose: they must hash the same after a round trip.
+const TEXTS: [&str; 6] = [
+    "plain text",
+    "ünïcödé — 日本語 🙂",
+    "",
+    "trailing spaces   ",
+    "tab\tand\nnewline",
+    "x",
+];
+
+fn signed_store(dir: &TempDir, float16: bool, k: &SigningKey) -> std::path::PathBuf {
+    let mut cfg = MemoryConfig::new(dir.path().join("s.h5"), "agent", DIM);
+    cfg.float16 = float16;
+    let path = cfg.path.clone();
+    let mut m = HDF5Memory::create(cfg).unwrap();
+    m.set_signing_key(k.clone());
+    let entries = (0..30).map(|i| entry(i, TEXTS[i % TEXTS.len()])).collect();
+    m.save_batch(entries).unwrap();
+    // Some graph and a deleted record, so every part of the manifest is used.
+    let a = m.knowledge_mut().add_entity("Alice", "person", 0);
+    let b = m.knowledge_mut().add_entity("Acme", "org", -1);
+    m.knowledge_mut().add_relation(a, b, "works_at", 0.75);
+    m.sessions_mut()
+        .add_at("s0", 0, 9, "chat", "first session", 1_700_000_000.0);
+    m.delete(4).unwrap();
+    m.flush_wal().unwrap();
+    path
+}
+
+fn verify(path: &Path, k: &SigningKey) -> VerifyReport {
+    HDF5Memory::verify(path, &k.verifying_key()).unwrap()
+}
+
+#[test]
+fn a_signed_store_verifies_through_reopen_and_checkpoint_cycles() {
+    for float16 in [true, false] {
+        let dir = TempDir::new().unwrap();
+        let k = key(7);
+        let path = signed_store(&dir, float16, &k);
+        let r = verify(&path, &k);
+        assert!(r.is_valid(), "float16={float16}: {r:?}");
+        assert_eq!(r.public_key, Some(k.verifying_key().to_bytes()));
+        assert_eq!(r.record_count, 30);
+        assert!(r.changed_records.is_empty());
+
+        // Reopen, change nothing, checkpoint again (with the key): still valid.
+        for _ in 0..3 {
+            let mut m = HDF5Memory::open(&path).unwrap();
+            assert!(m.is_signed());
+            m.set_signing_key(k.clone());
+            m.flush_wal().unwrap();
+            drop(m);
+            assert!(verify(&path, &k).is_valid());
+        }
+        // And after real changes, re-signed.
+        let mut m = HDF5Memory::open(&path).unwrap();
+        m.set_signing_key(k.clone());
+        m.save(entry(99, "added later")).unwrap();
+        m.hybrid_search(&entry(1, "").embedding, "text", 0.4, 0.6, 5);
+        m.flush_wal().unwrap();
+        drop(m);
+        let r = verify(&path, &k);
+        assert!(r.is_valid(), "{r:?}");
+        assert_eq!(r.record_count, 31);
+    }
+}
+
+#[test]
+fn a_signed_store_refuses_to_checkpoint_without_its_key() {
+    let dir = TempDir::new().unwrap();
+    let k = key(1);
+    let path = signed_store(&dir, true, &k);
+
+    let mut m = HDF5Memory::open(&path).unwrap();
+    m.save(entry(50, "pending")).unwrap();
+    match m.flush_wal() {
+        Err(MemoryError::SigningKeyRequired(msg)) => assert!(msg.contains("signed"), "{msg}"),
+        other => panic!("expected SigningKeyRequired, got {other:?}"),
+    }
+    // The file is untouched and still valid; the save is still in the WAL.
+    let r = verify(&path, &k);
+    assert!(r.is_valid());
+    assert_eq!(r.wal_entries_unsigned, 1);
+
+    // Supplying the key lets the checkpoint through, signed.
+    m.set_signing_key(k.clone());
+    m.flush_wal().unwrap();
+    drop(m);
+    let r = verify(&path, &k);
+    assert!(r.is_valid());
+    assert_eq!((r.record_count, r.wal_entries_unsigned), (31, 0));
+
+    // Removing the signature on purpose writes it unsigned.
+    let mut m = HDF5Memory::open(&path).unwrap();
+    m.remove_signature();
+    m.flush_wal().unwrap();
+    drop(m);
+    let r = verify(&path, &k);
+    assert!(!r.signed && !r.is_valid());
+    assert!(!HDF5Memory::open(&path).unwrap().is_signed());
+}
+
+#[test]
+fn the_wrong_key_does_not_verify_and_a_new_key_re_signs() {
+    let dir = TempDir::new().unwrap();
+    let (a, b) = (key(1), key(2));
+    let path = signed_store(&dir, true, &a);
+    let r = verify(&path, &b);
+    assert!(r.signed && !r.key_matches && !r.signature_valid && !r.is_valid());
+
+    let mut m = HDF5Memory::open(&path).unwrap();
+    m.set_signing_key(b.clone());
+    m.flush_wal().unwrap();
+    drop(m);
+    assert!(verify(&path, &b).is_valid());
+    assert!(!verify(&path, &a).is_valid());
+}
+
+/// Rewrite the store with changed contents but the *old* signature — what
+/// someone with write access to the file, but not the key, can do.
+fn tamper(path: &Path, change: impl FnOnce(&mut Tampered)) {
+    let file = clawhdf5::File::open(path).unwrap();
+    let (config, cache, sessions, knowledge) = schema::validate_and_load(&file).unwrap();
+    let checkpoint = schema::read_checkpoint_meta(&file);
+    let signature = schema::read_signature(&file).unwrap().unwrap();
+    drop(file);
+    let mut t = Tampered {
+        config,
+        cache,
+        sessions,
+        knowledge,
+    };
+    change(&mut t);
+    storage::write_to_disk_signed(
+        path,
+        &t.config,
+        &t.cache,
+        &t.sessions,
+        &t.knowledge,
+        &checkpoint,
+        Some(&signature),
+    )
+    .unwrap();
+}
+
+struct Tampered {
+    config: MemoryConfig,
+    cache: clawhdf5_agent::cache::MemoryCache,
+    sessions: clawhdf5_agent::SessionCache,
+    knowledge: clawhdf5_agent::knowledge::KnowledgeCache,
+}
+
+#[test]
+fn every_kind_of_edit_is_detected_and_located() {
+    let k = key(3);
+    type Edit = Box<dyn FnOnce(&mut Tampered)>;
+    type Case = (&'static str, Edit, fn(&VerifyReport) -> bool);
+    let cases: Vec<Case> = vec![
+        (
+            "record text",
+            Box::new(|t: &mut Tampered| t.cache.chunks[7] = "rewritten".into()),
+            |r| !r.records_match && r.changed_records == vec![7],
+        ),
+        (
+            "one embedding value",
+            Box::new(|t: &mut Tampered| {
+                let mut e = t.cache.embeddings[12].to_vec();
+                e[3] = 0.5;
+                t.cache.embeddings.set(12, &e);
+            }),
+            |r| r.changed_records == vec![12],
+        ),
+        (
+            "undelete",
+            Box::new(|t: &mut Tampered| t.cache.tombstones[4] = 0),
+            |r| r.changed_records == vec![4],
+        ),
+        (
+            "timestamp",
+            Box::new(|t: &mut Tampered| t.cache.timestamps[20] += 1.0),
+            |r| r.changed_records == vec![20],
+        ),
+        (
+            "record appended",
+            Box::new(|t: &mut Tampered| {
+                t.cache.push(
+                    "new".into(),
+                    vec![0.1; DIM],
+                    "x".into(),
+                    1.0,
+                    "s".into(),
+                    "".into(),
+                );
+            }),
+            |r| !r.records_match && r.changed_records == vec![30] && r.record_count == 31,
+        ),
+        (
+            "setting",
+            Box::new(|t: &mut Tampered| t.config.agent_id = "someone-else".into()),
+            |r| !r.settings_match && r.records_match,
+        ),
+        (
+            "session summary",
+            Box::new(|t: &mut Tampered| t.sessions.summaries[0] = "edited".into()),
+            |r| !r.sessions_match && r.records_match,
+        ),
+        (
+            "graph edge",
+            Box::new(|t: &mut Tampered| t.knowledge.relations[0].weight = 1.0),
+            |r| !r.graph_match && r.records_match,
+        ),
+    ];
+    for (name, edit, check) in cases {
+        let dir = TempDir::new().unwrap();
+        let path = signed_store(&dir, true, &k);
+        tamper(&path, edit);
+        let r = verify(&path, &k);
+        assert!(
+            r.signed && r.key_matches && r.signature_valid,
+            "{name}: {r:?}"
+        );
+        assert!(!r.is_valid(), "{name}: edit not detected: {r:?}");
+        assert!(check(&r), "{name}: {r:?}");
+    }
+}
+
+#[test]
+fn a_forged_manifest_fails_the_signature() {
+    // Recomputing the hashes for tampered contents does not help without the
+    // key: the signature no longer matches the manifest.
+    let dir = TempDir::new().unwrap();
+    let k = key(5);
+    let path = signed_store(&dir, true, &k);
+    let file = clawhdf5::File::open(&path).unwrap();
+    let (config, mut cache, sessions, knowledge) = schema::validate_and_load(&file).unwrap();
+    let checkpoint = schema::read_checkpoint_meta(&file);
+    let mut sig = schema::read_signature(&file).unwrap().unwrap();
+    drop(file);
+    cache.chunks[0] = "forged".into();
+    // Re-sign with an attacker key, then splice the victim's public key back.
+    let forged = clawhdf5_agent::signing::sign(
+        &key(66),
+        &config,
+        &cache,
+        &sessions,
+        &knowledge,
+        checkpoint.wal_applied,
+    );
+    sig.manifest = forged.manifest;
+    sig.record_hashes = forged.record_hashes;
+    storage::write_to_disk_signed(
+        &path,
+        &config,
+        &cache,
+        &sessions,
+        &knowledge,
+        &checkpoint,
+        Some(&sig),
+    )
+    .unwrap();
+    let r = verify(&path, &k);
+    assert!(
+        r.key_matches && !r.signature_valid && !r.is_valid(),
+        "{r:?}"
+    );
+}
+
+#[test]
+fn an_unsigned_store_reports_unsigned() {
+    let dir = TempDir::new().unwrap();
+    let mut m = HDF5Memory::create(MemoryConfig::new(dir.path().join("u.h5"), "a", DIM)).unwrap();
+    m.save_batch(vec![entry(0, "hello")]).unwrap();
+    drop(m);
+    let r = HDF5Memory::verify(&dir.path().join("u.h5"), &VerifyingKey::from(&key(1))).unwrap();
+    assert!(!r.signed && !r.is_valid());
+    assert_eq!(r.record_count, 1);
+}
+
+#[test]
+fn nul_bytes_in_text_still_verify() {
+    // Strings are stored null-padded; the hash must follow what a reopened
+    // store actually holds, or an untouched store would fail to verify.
+    let dir = TempDir::new().unwrap();
+    let k = key(9);
+    let mut m = HDF5Memory::create(MemoryConfig::new(dir.path().join("n.h5"), "a", DIM)).unwrap();
+    m.set_signing_key(k.clone());
+    m.save_batch(vec![
+        entry(0, "inner\0nul"),
+        entry(1, "trailing nul\0"),
+        entry(2, "\0leading"),
+    ])
+    .unwrap();
+    drop(m);
+    let r = verify(&dir.path().join("n.h5"), &k);
+    assert!(r.is_valid(), "{r:?}");
+    let m = HDF5Memory::open(&dir.path().join("n.h5")).unwrap();
+    eprintln!(
+        "reloaded: {:?}",
+        (0..3).map(|i| m.get_chunk(i)).collect::<Vec<_>>()
+    );
+}

crates/clawhdf5-android/Cargo.toml

@@ -2,6 +2,7 @@
 name = "clawhdf5-android"
 version = "2.7.0"
 edition = "2024"
+rust-version.workspace = true
 description = "Android JNI bridge for edgehdf5-memory HDF5 backend"
 license = "MIT"
 

crates/clawhdf5-ann/Cargo.toml

@@ -2,6 +2,7 @@
 name = "clawhdf5-ann"
 version = "2.7.0"
 edition = "2024"
+rust-version.workspace = true
 description = "HNSW approximate nearest neighbor index stored as HDF5"
 license = "MIT"
 repository = "https://git.redclaw.dev/quantumclaw/clawhdf5"

crates/clawhdf5-bench/Cargo.toml

@@ -2,6 +2,7 @@
 name = "clawhdf5-bench"
 version = "2.7.0"
 edition = "2024"
+rust-version.workspace = true
 description = "Benchmark harnesses for clawhdf5-agent (Track 8)"
 license = "MIT"
 

crates/clawhdf5-bench/src/bin/consolidation_efficiency.rs

@@ -396,7 +396,7 @@ fn run_memory_reduction_benchmark() {
     println!();
     println!(
         "{:>8}  {:>10}  {:>10}  {:>10}  {:>12}",
-        "Initial", "Remaining", "Eviction%", "Signal OK?", "BM25 Speedup"
+        "Initial", "Remaining", "Eviction%", "Signal OK?", "Records ÷"
     );
     println!("{}", "-".repeat(58));
 
@@ -440,7 +440,8 @@ fn run_memory_reduction_benchmark() {
         // Check all signal records survived
         let signal_survived = signal_ids.iter().all(|&id| engine.get_by_id(id).is_some());
 
-        // Rough speedup: BM25 scales roughly linearly with record count
+        // How many times fewer records there are. Not a measured speedup —
+        // Part 1 measures search latency before and after.
         let speedup = before_count as f64 / after_count.max(1) as f64;
 
         println!(
@@ -480,7 +481,7 @@ fn main() {
     println!("  3. Reducing search latency proportional to record reduction");
     println!();
     println!(
-        "Cycle time scales sub-linearly: 100 records ~microseconds, 100K records ~tens of ms."
+        "Cycle time grows a little faster than linearly: 100 records ~microseconds, 100K records ~tens of ms."
     );
     println!("Signal records with Correction source + high access_count survive eviction.");
 }

crates/clawhdf5-bench/src/bin/footprint_bench.rs

@@ -11,12 +11,14 @@
 //!
 //! Configuration matrix:
 //!   - Text lengths: short (50 chars), medium (200 chars), long (1000 chars)
-//!   - Embedding: 384-dim f32 (1536 bytes raw per record)
+//!   - Embedding: 384-dim, stored as float16 (the default for new stores) or
+//!     f32 with `--f32`; "raw" bytes are counted as f32 input either way
 //!   - WAL: enabled and disabled
 //!
 //! # Usage
 //! ```
-//! cargo run --release --bin footprint_bench
+//! cargo run --release --bin footprint_bench           # float16 stores
+//! cargo run --release --bin footprint_bench -- --f32  # f32 stores
 //! ```
 
 use std::time::Instant;
@@ -24,6 +26,9 @@ use std::time::Instant;
 use clawhdf5_agent::{AgentMemory, HDF5Memory, MemoryConfig, MemoryEntry};
 use tempfile::TempDir;
 
+/// `--f32`: build f32 stores instead of the library's float16 default.
+static F32: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);
+
 const EMBEDDING_DIM: usize = 384;
 
 // Raw bytes per record: 384 f32 embeddings + median text + overhead
@@ -152,6 +157,9 @@ fn measure_footprint(
     config.compression = compression;
     config.compression_level = if compression { 6 } else { 0 };
     config.compact_threshold = 0.0;
+    if F32.load(std::sync::atomic::Ordering::Relaxed) {
+        config.float16 = false;
+    }
 
     let mut memory = HDF5Memory::create(config).expect("HDF5Memory::create failed");
 
@@ -241,11 +249,19 @@ fn fmt_n(n: usize) -> String {
 // ---------------------------------------------------------------------------
 
 fn main() {
+    if std::env::args().skip(1).any(|a| a == "--f32") {
+        F32.store(true, std::sync::atomic::Ordering::Relaxed);
+    }
+    let stored = if F32.load(std::sync::atomic::Ordering::Relaxed) {
+        "f32 (1,536 bytes per record)"
+    } else {
+        "float16 (768 bytes per record; the default for new stores)"
+    };
     println!("=================================================================");
     println!("  ClawhDF5 Memory Footprint Benchmark");
     println!("=================================================================");
     println!();
-    println!("Embedding: 384-dim f32 = 1,536 bytes raw per record");
+    println!("Embedding: 384-dim, stored as {stored}; raw input counted as f32");
     println!("Text lengths: short=50 chars, medium=200 chars, long=1000 chars");
     println!();
 

crates/clawhdf5-bench/src/bin/longmemeval_bench.rs

@@ -64,6 +64,11 @@ use tempfile::TempDir;
 
 const EMBEDDING_DIM: usize = 384;
 
+/// `--float16`: build every per-question store with `MemoryConfig::float16`,
+/// so embeddings are rounded to half precision as they are saved — exactly
+/// what such a store searches over.
+static FLOAT16: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);
+
 /// A mode's fusion, as one short string for the reports.
 fn describe(mode: Mode) -> String {
     let fusion = match mode.fusion {
@@ -431,6 +436,7 @@ fn evaluate_question(
     let mut config = MemoryConfig::new(dir.path().join("lme.h5"), "lme-bench", EMBEDDING_DIM);
     config.wal_enabled = false;
     config.compact_threshold = 0.0;
+    config.float16 = FLOAT16.load(std::sync::atomic::Ordering::Relaxed);
 
     let mut memory = HDF5Memory::create(config).expect("failed to create HDF5Memory");
     memory.set_token_filter(mode.tokens);
@@ -940,6 +946,10 @@ fn main() {
                 limit = Some(v.parse().expect("--limit must be a positive integer"));
             }
             "--sweep" => sweep = true,
+            "--float16" => {
+                FLOAT16.store(true, std::sync::atomic::Ordering::Relaxed);
+                eprintln!("Stores use MemoryConfig::float16 (half-precision embeddings)");
+            }
             "--rerank-sweep" => {
                 // Re-ranking needs the vector stage to have candidates worth
                 // reordering, so this is an embeddings-only comparison.
@@ -971,6 +981,9 @@ fn main() {
                      --rerank-sweep\n\
                               compare re-ranking off, metadata-only (the old\n\
                               behaviour) and blended at several half-lives.\n\
+                     --float16\n\
+                              build each store with MemoryConfig::float16, to\n\
+                              compare retrieval on half-precision embeddings.\n\
                      --sweep  instead of the three named modes, sweep vector_weight\n\
                               from 0.0 to 1.0 in 0.1 steps. The 0.7/0.3 default was\n\
                               never searched; this is what searches it."

crates/clawhdf5-bench/src/bin/search_harness.rs

@@ -19,6 +19,9 @@
 //! cargo run --release -p clawhdf5-bench --bin search_harness -- --full  # + 100K
 //! cargo run --release -p clawhdf5-bench --bin search_harness -- --json out.json
 //! cargo run --release -p clawhdf5-bench --bin search_harness -- --ann-only --uniform
+//! cargo run --release -p clawhdf5-bench --bin search_harness -- --float16-study --full
+//! cargo run --release -p clawhdf5-bench --bin search_harness -- --options-study --full
+//! cargo run --release -p clawhdf5-bench --bin search_harness -- --signing-study --full
 //! ```
 
 use std::time::{Duration, Instant};
@@ -88,6 +91,9 @@ static UNIFORM: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::n
 /// the memory) instead of f32, to price the recall it costs.
 static INT8: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);
 
+/// `--f16-first`: in `--float16-study`, run the float16 store first.
+static F16_FIRST: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);
+
 /// `--rerank`: re-score the candidate pool against the exact vectors before
 /// taking the top K.
 static RERANK: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);
@@ -483,6 +489,394 @@ fn bench_end_to_end(n: usize, json: &mut Vec<serde_json::Value>) {
     }));
 }
 
+// ---------------------------------------------------------------------------
+// Signing study: what does an Ed25519-signed checkpoint cost?
+// ---------------------------------------------------------------------------
+
+/// `--signing-study`: checkpoint time unsigned vs signed, `verify` time, and
+/// the file-size cost of the stored per-record hashes. Default store
+/// settings (float16, int8 index). Medians of five checkpoints / three
+/// verifies.
+fn signing_study(n: usize) {
+    use clawhdf5_agent::signing::SigningKey;
+    let data = make_dataset(n, 0x516 ^ n as u64);
+    let mut rng = Rng(9);
+    let entries: Vec<MemoryEntry> = data
+        .vectors
+        .iter()
+        .enumerate()
+        .map(|(i, v)| MemoryEntry {
+            chunk: text_for(data.cluster_of[i], i, &mut rng),
+            embedding: v.clone(),
+            source_channel: "bench".into(),
+            timestamp: i as f64,
+            session_id: format!("s{}", i % 50),
+            tags: format!("t{i}"),
+        })
+        .collect();
+    let dir = tempfile::tempdir().unwrap();
+    let path = dir.path().join("sign.h5");
+    let mut mem = HDF5Memory::create(MemoryConfig::new(path.clone(), "bench", DIM)).unwrap();
+    mem.save_batch(entries).unwrap();
+    std::hint::black_box(mem.hybrid_search(&data.queries[0], "", 1.0, 0.0, K));
+
+    let median = |mut v: Vec<Duration>| {
+        v.sort();
+        v[v.len() / 2]
+    };
+    let checkpoint = |mem: &mut HDF5Memory| {
+        median(
+            (0..5)
+                .map(|_| {
+                    let t = Instant::now();
+                    mem.flush_wal().unwrap();
+                    t.elapsed()
+                })
+                .collect(),
+        )
+    };
+    let unsigned = checkpoint(&mut mem);
+    let unsigned_bytes = std::fs::metadata(&path).unwrap().len();
+    let key = SigningKey::from_bytes(&[7; 32]);
+    mem.set_signing_key(key.clone());
+    let signed = checkpoint(&mut mem);
+    let signed_bytes = std::fs::metadata(&path).unwrap().len();
+    drop(mem);
+    let vk = key.verifying_key();
+    let verify = median(
+        (0..3)
+            .map(|_| {
+                let t = Instant::now();
+                let r = HDF5Memory::verify(&path, &vk).unwrap();
+                let d = t.elapsed();
+                assert!(r.is_valid());
+                d
+            })
+            .collect(),
+    );
+    println!(
+        "| {n} | {:.1} | {:.1} | {:+.1} | {:.1} | {:+.2} |",
+        millis(unsigned),
+        millis(signed),
+        millis(signed) - millis(unsigned),
+        millis(verify),
+        (signed_bytes as f64 - unsigned_bytes as f64) / (1024.0 * 1024.0),
+    );
+}
+
+// ---------------------------------------------------------------------------
+// Search options study: source filters, re-ranking, confidence rejection
+// ---------------------------------------------------------------------------
+
+/// `--options-study`: what `HDF5Memory::search`'s options cost and whether a
+/// filtered search finds the right records. Filters keep 50%, 10% or 1% of
+/// the store at random, or two whole clusters away from the query (the case
+/// the index cannot serve, which falls back to an exact scan). Recall is
+/// vector-only against an exact scan of the allowed records; latency is full
+/// hybrid search. Hebbian boosting is off.
+fn options_study(n: usize) {
+    use clawhdf5_agent::SearchOptions;
+    use clawhdf5_agent::confidence::ConfidenceConfig;
+    use clawhdf5_agent::hybrid::Fusion;
+    use clawhdf5_agent::reranker::ReRankConfig;
+
+    let data = make_dataset(n, 0x0B7 ^ n as u64);
+    let n_clusters = data.cluster_of.iter().max().map_or(1, |m| m + 1);
+    let mut rng = Rng(5);
+    let bucket_of: Vec<usize> = (0..n).map(|_| rng.below(100)).collect();
+    let bucket = &bucket_of;
+    let query_texts: Vec<String> = data
+        .query_cluster
+        .iter()
+        .enumerate()
+        .map(|(i, c)| text_for(*c, i, &mut rng))
+        .collect();
+    let exact_top = |q: &[f32], allowed: &dyn Fn(usize) -> bool| -> Vec<usize> {
+        let mut s: Vec<(usize, f32)> = (0..n)
+            .filter(|&i| allowed(i))
+            .map(|i| (i, data.vectors[i].iter().zip(q).map(|(a, b)| a * b).sum()))
+            .collect();
+        s.sort_by(|a, b| b.1.total_cmp(&a.1).then(a.0.cmp(&b.0)));
+        s.into_iter().take(K).map(|(i, _)| i).collect()
+    };
+
+    // Two stores: channel = random bucket, and channel = cluster.
+    let dir = tempfile::tempdir().unwrap();
+    let mut stores = Vec::new();
+    for by_cluster in [false, true] {
+        let mut rng = Rng(3);
+        let entries: Vec<MemoryEntry> = data
+            .vectors
+            .iter()
+            .enumerate()
+            .map(|(i, v)| MemoryEntry {
+                chunk: text_for(data.cluster_of[i], i, &mut rng),
+                embedding: v.clone(),
+                source_channel: if by_cluster {
+                    format!("c{}", data.cluster_of[i])
+                } else {
+                    format!("b{}", bucket[i])
+                },
+                timestamp: i as f64,
+                session_id: format!("s{}", i % 50),
+                tags: format!("t{i}"),
+            })
+            .collect();
+        let mut config = MemoryConfig::new(
+            dir.path().join(format!("opt_{by_cluster}.h5")),
+            "bench",
+            DIM,
+        );
+        config.hebbian_boost = 0.0;
+        let mut mem = HDF5Memory::create(config).unwrap();
+        mem.save_batch(entries).unwrap();
+        std::hint::black_box(mem.search(&data.queries[0], "", &SearchOptions::new(K)));
+        stores.push(mem);
+    }
+
+    let vector_only = SearchOptions::new(K).with_fusion(Fusion::Weighted {
+        vector: 1.0,
+        keyword: 0.0,
+    });
+    // (label, store, channels for query i, allowed(i, record))
+    type Case<'a> = (
+        String,
+        usize,
+        Box<dyn Fn(usize) -> Option<Vec<String>> + 'a>,
+        Box<dyn Fn(usize, usize) -> bool + 'a>,
+    );
+    let mut cases: Vec<Case> = vec![(
+        "no filter".into(),
+        0,
+        Box::new(|_| None),
+        Box::new(|_, _| true),
+    )];
+    for pct in [50usize, 10, 1] {
+        cases.push((
+            format!("random {pct}%"),
+            0,
+            Box::new(move |_| Some((0..pct).map(|b| format!("b{b}")).collect())),
+            Box::new(move |_, i| bucket[i] < pct),
+        ));
+    }
+    let d = &data;
+    let away = move |qi: usize| {
+        let qc = d.query_cluster[qi];
+        [
+            (qc + n_clusters / 3) % n_clusters,
+            (qc + 2 * n_clusters / 3) % n_clusters,
+        ]
+    };
+    cases.push((
+        "2 clusters away from the query".into(),
+        1,
+        Box::new(move |qi| Some(away(qi).iter().map(|c| format!("c{c}")).collect())),
+        Box::new(move |qi, i| away(qi).contains(&d.cluster_of[i])),
+    ));
+
+    for (label, store, channels, allowed) in &cases {
+        let mem = &mut stores[*store];
+        let mut hits = 0;
+        let mut kept = 0;
+        for (qi, q) in data.queries.iter().enumerate() {
+            let mut opts = vector_only.clone();
+            opts.source_channels = channels(qi);
+            let got = mem.search(q, "", &opts);
+            let want = exact_top(q, &|i| allowed(qi, i));
+            kept += want.len();
+            hits += got.iter().filter(|r| want.contains(&r.index)).count();
+        }
+        let latency = summarize(
+            (0..N_QUERIES)
+                .map(|qi| {
+                    let mut opts = SearchOptions::new(K);
+                    opts.source_channels = channels(qi);
+                    let t = Instant::now();
+                    std::hint::black_box(mem.search(&data.queries[qi], &query_texts[qi], &opts));
+                    t.elapsed()
+                })
+                .collect(),
+        );
+        println!(
+            "| {n} | {label} | {:.4} | {:.3} | {:.3} |",
+            hits as f64 / kept.max(1) as f64,
+            millis(latency.p50),
+            millis(latency.p99),
+        );
+    }
+
+    let mem = &mut stores[0];
+    for (label, opts) in [
+        (
+            "re-rank",
+            SearchOptions::new(K).with_rerank(ReRankConfig::default()),
+        ),
+        (
+            "re-rank + confidence",
+            SearchOptions::new(K)
+                .with_rerank(ReRankConfig::default())
+                .with_confidence(ConfidenceConfig::default()),
+        ),
+    ] {
+        let latency = summarize(
+            (0..N_QUERIES)
+                .map(|qi| {
+                    let t = Instant::now();
+                    std::hint::black_box(mem.search(&data.queries[qi], &query_texts[qi], &opts));
+                    t.elapsed()
+                })
+                .collect(),
+        );
+        println!(
+            "| {n} | {label} | — | {:.3} | {:.3} |",
+            millis(latency.p50),
+            millis(latency.p99)
+        );
+    }
+}
+
+// ---------------------------------------------------------------------------
+// float16 study: what does half-precision embedding storage cost?
+// ---------------------------------------------------------------------------
+
+/// `--float16-study`: the same data in an `f32` store and a `float16` store.
+/// Reports file size, checkpoint and open time, vector-search recall@10
+/// against an exact scan of the *original* f32 vectors, how often the two
+/// stores return the same top 10, and `hybrid_search` latency. Hebbian
+/// boosting is off, so every query sees the same store.
+fn float16_study(n: usize) {
+    let data = make_dataset(n, 0xF16 ^ n as u64);
+    let mut rng = Rng(11);
+    let query_texts: Vec<String> = data
+        .query_cluster
+        .iter()
+        .enumerate()
+        .map(|(i, c)| text_for(*c, i, &mut rng))
+        .collect();
+
+    // Exact top K by cosine (the vectors are unit length) on the f32 inputs.
+    let exact: Vec<Vec<usize>> = data
+        .queries
+        .iter()
+        .map(|q| {
+            let mut scored: Vec<(usize, f32)> = data
+                .vectors
+                .iter()
+                .enumerate()
+                .map(|(i, v)| (i, v.iter().zip(q).map(|(a, b)| a * b).sum()))
+                .collect();
+            scored.sort_by(|a, b| b.1.total_cmp(&a.1).then(a.0.cmp(&b.0)));
+            scored.into_iter().take(K).map(|(i, _)| i).collect()
+        })
+        .collect();
+
+    let dir = tempfile::tempdir().unwrap();
+    let mut per_variant: Vec<(bool, Vec<Vec<usize>>)> = Vec::new();
+    // `--f16-first` swaps the order, to check the numbers do not depend on
+    // which store runs first (page cache, allocator, CPU frequency).
+    let order = if F16_FIRST.load(std::sync::atomic::Ordering::Relaxed) {
+        [true, false]
+    } else {
+        [false, true]
+    };
+    for float16 in order {
+        let path = dir.path().join(format!("f16study_{float16}.h5"));
+        let mut rng = Rng(3);
+        let entries: Vec<MemoryEntry> = data
+            .vectors
+            .iter()
+            .enumerate()
+            .map(|(i, v)| MemoryEntry {
+                chunk: text_for(data.cluster_of[i], i, &mut rng),
+                embedding: v.clone(),
+                source_channel: "bench".into(),
+                timestamp: i as f64,
+                session_id: format!("s{}", i % 50),
+                tags: format!("t{i}"),
+            })
+            .collect();
+        let mut config = MemoryConfig::new(path.clone(), "bench", DIM);
+        config.float16 = float16;
+        config.hebbian_boost = 0.0;
+        let mut mem = HDF5Memory::create(config).unwrap();
+        mem.save_batch(entries).unwrap();
+        // Build the indexes, then time a checkpoint that writes everything.
+        std::hint::black_box(mem.hybrid_search(&data.queries[0], "", 1.0, 0.0, K));
+        let t = Instant::now();
+        mem.flush_wal().unwrap();
+        let checkpoint = t.elapsed();
+        drop(mem);
+        let file_bytes = std::fs::metadata(&path).unwrap().len();
+
+        // Median of three opens.
+        let mut opens: Vec<Duration> = (0..3)
+            .map(|_| {
+                let t = Instant::now();
+                let m = HDF5Memory::open(&path).unwrap();
+                let d = t.elapsed();
+                drop(m);
+                d
+            })
+            .collect();
+        opens.sort();
+        let mut mem = HDF5Memory::open(&path).unwrap();
+
+        // Vector-only search: empty text, all weight on the vector stage.
+        let results: Vec<Vec<usize>> = data
+            .queries
+            .iter()
+            .map(|q| {
+                mem.hybrid_search(q, "", 1.0, 0.0, K)
+                    .iter()
+                    .map(|r| r.index)
+                    .collect()
+            })
+            .collect();
+        let hits: usize = results
+            .iter()
+            .zip(&exact)
+            .map(|(got, want)| got.iter().filter(|i| want.contains(i)).count())
+            .sum();
+        let recall = hits as f64 / (K * data.queries.len()) as f64;
+
+        let latency = summarize(
+            (0..N_QUERIES)
+                .map(|i| {
+                    let t = Instant::now();
+                    std::hint::black_box(mem.hybrid_search(
+                        &data.queries[i],
+                        &query_texts[i],
+                        0.4,
+                        0.6,
+                        K,
+                    ));
+                    t.elapsed()
+                })
+                .collect(),
+        );
+        let overlap = match per_variant.first() {
+            Some((_, other)) => {
+                let same: usize = results
+                    .iter()
+                    .zip(other)
+                    .map(|(a, b)| a.iter().filter(|i| b.contains(i)).count())
+                    .sum();
+                format!("{:.4}", same as f64 / (K * data.queries.len()) as f64)
+            }
+            None => "—".into(),
+        };
+        println!(
+            "| {n} | {} | {:.1} | {:.0} | {:.1} | {recall:.4} | {overlap} | {:.3} |",
+            if float16 { "float16" } else { "f32" },
+            mib(file_bytes),
+            millis(checkpoint),
+            millis(opens[1]),
+            millis(latency.p50),
+        );
+        per_variant.push((float16, results));
+    }
+}
+
 // ---------------------------------------------------------------------------
 // Fusion study: does capping the keyword candidate pool change the ranking?
 // ---------------------------------------------------------------------------
@@ -642,6 +1036,52 @@ fn main() {
         }
         return;
     }
+    if args.iter().any(|a| a == "--signing-study") {
+        println!("## Signed checkpoints ({DIM}-dim, float16, int8 index)\n");
+        println!(
+            "| N | checkpoint ms, unsigned | checkpoint ms, signed | signing adds ms | verify ms | file MiB added |"
+        );
+        println!("|---:|---:|---:|---:|---:|---:|");
+        for &n in if full {
+            &[1_000, 10_000, 100_000][..]
+        } else {
+            &[1_000, 10_000][..]
+        } {
+            signing_study(n);
+        }
+        return;
+    }
+    if args.iter().any(|a| a == "--options-study") {
+        println!("## Search options ({DIM}-dim, k = {K}, Hebbian boost off)\n");
+        println!("| N | options | filtered recall@10 | p50 ms | p99 ms |");
+        println!("|---:|---|---:|---:|---:|");
+        for &n in if full {
+            &[10_000, 100_000][..]
+        } else {
+            &[10_000][..]
+        } {
+            options_study(n);
+        }
+        return;
+    }
+    if args.iter().any(|a| a == "--f16-first") {
+        F16_FIRST.store(true, std::sync::atomic::Ordering::Relaxed);
+    }
+    if args.iter().any(|a| a == "--float16-study") {
+        println!("## float16 embedding storage ({DIM}-dim, int8 index, Hebbian boost off)\n");
+        println!(
+            "| N | embeddings | file MiB | checkpoint ms | open ms | recall@10 | top-10 overlap with the other | hybrid p50 ms |"
+        );
+        println!("|---:|---|---:|---:|---:|---:|---:|---:|");
+        for &n in if full {
+            &[1_000, 10_000, 100_000][..]
+        } else {
+            &[1_000, 10_000][..]
+        } {
+            float16_study(n);
+        }
+        return;
+    }
     if args.iter().any(|a| a == "--int8") {
         INT8.store(true, std::sync::atomic::Ordering::Relaxed);
         println!("(int8-quantised index vectors)");

crates/clawhdf5-cli/Cargo.toml

@@ -2,6 +2,7 @@
 name = "clawhdf5-cli"
 version = "2.7.0"
 edition = "2024"
+rust-version.workspace = true
 license = "MIT"
 description = "CLI for clawhdf5 agent memory — create, save, search, recall, stats"
 repository = "https://git.redclaw.dev/quantumclaw/clawhdf5"

crates/clawhdf5-cli/src/main.rs

@@ -1,15 +1,22 @@
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 
 use clap::{Parser, Subcommand};
+use clawhdf5_agent::signing::{self, SigningKey, VerifyingKey};
 use clawhdf5_agent::{AgentMemory, HDF5Memory, MemoryConfig, MemoryEntry};
 
 /// ClawhDF5 — HDF5-backed cognitive memory for AI agents
 #[derive(Parser)]
 #[command(name = "clawhdf5", version, about)]
 struct Cli {
-    /// Path to the .h5 memory file
+    /// Path to the .h5 memory file (not needed for `keygen`)
     #[arg(short, long, env = "CLAWHDF5_PATH")]
-    path: PathBuf,
+    path: Option<PathBuf>,
+
+    /// File holding an Ed25519 signing key (64 hex characters, from
+    /// `keygen`). Every checkpoint this command makes is then signed; a
+    /// signed store refuses to checkpoint without it.
+    #[arg(long, env = "CLAWHDF5_SIGNING_KEY", global = true)]
+    signing_key: Option<PathBuf>,
 
     #[command(subcommand)]
     command: Commands,
@@ -28,10 +35,22 @@ enum Commands {
         /// Enable write-ahead log
         #[arg(long)]
         wal: bool,
-        /// Store the vector index's copy of the embeddings as int8, roughly
-        /// halving a loaded store's memory at about 13% fewer queries/second
+        /// Hold the vector index's copy of the embeddings as f32 instead of
+        /// the default int8 (which uses a quarter of the memory and is faster
+        /// at equal recall)
         #[arg(long)]
+        f32_index: bool,
+        /// Accepted for compatibility; int8 is now the default
+        #[arg(long, hide = true, conflicts_with = "f32_index")]
         quantized_index: bool,
+        /// Store embeddings as full-precision f32 instead of the default
+        /// half precision (float16: half the bytes, about three significant
+        /// digits, values within ±65504)
+        #[arg(long)]
+        f32: bool,
+        /// Accepted for compatibility; float16 is now the default
+        #[arg(long, hide = true, conflicts_with = "f32")]
+        float16: bool,
     },
     /// Save a memory entry (reads JSON from stdin or --json)
     Save {
@@ -79,6 +98,38 @@ enum Commands {
         /// Destination path
         dest: PathBuf,
     },
+    /// Generate an Ed25519 signing key for signed checkpoints
+    Keygen {
+        /// Where to write the secret key (created new, owner-only on Unix)
+        #[arg(long)]
+        out: PathBuf,
+    },
+    /// Verify a signed store against a public key; exit status 2 if not valid
+    Verify {
+        /// The trusted public key: 64 hex characters, or a file holding them
+        #[arg(long)]
+        public_key: String,
+    },
+}
+
+fn read_signing_key(path: &Path) -> Result<SigningKey, Box<dyn std::error::Error>> {
+    let text = std::fs::read_to_string(path)
+        .map_err(|e| format!("cannot read signing key {}: {e}", path.display()))?;
+    let bytes = signing::from_hex::<32>(&text)
+        .ok_or_else(|| format!("{} is not a 64-hex-character key", path.display()))?;
+    Ok(SigningKey::from_bytes(&bytes))
+}
+
+/// Open for writing, with the signing key applied if one was given.
+fn open_writable(
+    path: &Path,
+    key: &Option<SigningKey>,
+) -> Result<HDF5Memory, Box<dyn std::error::Error>> {
+    let mut mem = HDF5Memory::open(path)?;
+    if let Some(k) = key {
+        mem.set_signing_key(k.clone());
+    }
+    Ok(mem)
 }
 
 fn main() {
@@ -91,24 +142,76 @@ fn main() {
 }
 
 fn run(cli: Cli) -> Result<(), Box<dyn std::error::Error>> {
+    if let Commands::Keygen { out } = &cli.command {
+        let key = signing::generate_key();
+        let mut opts = std::fs::OpenOptions::new();
+        opts.write(true).create_new(true);
+        #[cfg(unix)]
+        {
+            use std::os::unix::fs::OpenOptionsExt;
+            opts.mode(0o600);
+        }
+        use std::io::Write;
+        let mut f = opts
+            .open(out)
+            .map_err(|e| format!("cannot create {}: {e}", out.display()))?;
+        writeln!(f, "{}", signing::to_hex(&key.to_bytes()))?;
+        let j = serde_json::json!({
+            "status": "generated",
+            "secret_key_file": out.display().to_string(),
+            "public_key": signing::to_hex(&key.verifying_key().to_bytes()),
+        });
+        println!("{}", serde_json::to_string_pretty(&j)?);
+        return Ok(());
+    }
+    let path = cli
+        .path
+        .clone()
+        .ok_or("--path (or CLAWHDF5_PATH) is required")?;
+    let key = cli
+        .signing_key
+        .as_deref()
+        .map(read_signing_key)
+        .transpose()?;
     match cli.command {
         Commands::Create {
             agent_id,
             dim,
             wal,
-            quantized_index,
+            f32_index,
+            quantized_index: _,
+            f32,
+            float16: _,
         } => {
-            let mut config = MemoryConfig::new(cli.path.clone(), &agent_id, dim);
+            let mut config = MemoryConfig::new(path.clone(), &agent_id, dim);
             config.wal_enabled = wal;
-            config.quantized_index = quantized_index;
-            let mem = HDF5Memory::create(config)?;
+            // As with --f32-index: only ever switch the library default off.
+            if f32 {
+                config.float16 = false;
+            }
+            let config_float16 = config.float16;
+            // Only ever switch *off* the library default: assigning the flag
+            // outright would force every CLI-created store back to f32 unless
+            // the caller knew to ask for int8.
+            if f32_index {
+                config.quantized_index = false;
+            }
+            let config_quantized = config.quantized_index;
+            let mut mem = HDF5Memory::create(config)?;
+            // Sign straight away, so the store is never on disk unsigned.
+            if let Some(k) = &key {
+                mem.set_signing_key(k.clone());
+                mem.flush_wal()?;
+            }
             let j = serde_json::json!({
                 "status": "created",
-                "path": cli.path.display().to_string(),
+                "path": path.display().to_string(),
                 "agent_id": agent_id,
                 "embedding_dim": dim,
                 "wal_enabled": wal,
-                "quantized_index": quantized_index,
+                "quantized_index": config_quantized,
+                "float16": config_float16,
+                "signed": mem.is_signed(),
                 "count": mem.count(),
             });
             println!("{}", serde_json::to_string_pretty(&j)?);
@@ -125,7 +228,7 @@ fn run(cli: Cli) -> Result<(), Box<dyn std::error::Error>> {
                 }
             };
             let entry: MemoryEntry = serde_json::from_str(&input)?;
-            let mut mem = HDF5Memory::open(&cli.path)?;
+            let mut mem = open_writable(&path, &key)?;
             let idx = mem.save(entry)?;
             let j = serde_json::json!({ "status": "saved", "index": idx, "count": mem.count() });
             println!("{}", serde_json::to_string(&j)?);
@@ -139,7 +242,7 @@ fn run(cli: Cli) -> Result<(), Box<dyn std::error::Error>> {
             keyword_weight,
         } => {
             let emb: Vec<f32> = serde_json::from_str(&embedding)?;
-            let mut mem = HDF5Memory::open(&cli.path)?;
+            let mut mem = open_writable(&path, &key)?;
             let results = mem.hybrid_search(&emb, &query, vector_weight, keyword_weight, top_k);
             let j: Vec<serde_json::Value> = results
                 .iter()
@@ -157,7 +260,7 @@ fn run(cli: Cli) -> Result<(), Box<dyn std::error::Error>> {
         }
 
         Commands::Recall { index } => {
-            let mem = HDF5Memory::open_read_only(&cli.path)?;
+            let mem = HDF5Memory::open_read_only(&path)?;
             match mem.get_chunk(index) {
                 Some(content) => {
                     let j = serde_json::json!({ "index": index, "chunk": content });
@@ -171,22 +274,23 @@ fn run(cli: Cli) -> Result<(), Box<dyn std::error::Error>> {
         }
 
         Commands::Stats => {
-            let mem = HDF5Memory::open_read_only(&cli.path)?;
+            let mem = HDF5Memory::open_read_only(&path)?;
             let cfg = mem.config();
             let j = serde_json::json!({
-                "path": cli.path.display().to_string(),
+                "path": path.display().to_string(),
                 "agent_id": cfg.agent_id,
                 "embedding_dim": cfg.embedding_dim,
                 "count": mem.count(),
                 "active": mem.count_active(),
                 "wal_enabled": cfg.wal_enabled,
                 "wal_pending": mem.wal_pending_count(),
+                "signed": mem.is_signed(),
             });
             println!("{}", serde_json::to_string_pretty(&j)?);
         }
 
         Commands::FlushWal => {
-            let mut mem = HDF5Memory::open(&cli.path)?;
+            let mut mem = open_writable(&path, &key)?;
             let before = mem.wal_pending_count();
             mem.flush_wal()?;
             let j = serde_json::json!({
@@ -198,7 +302,7 @@ fn run(cli: Cli) -> Result<(), Box<dyn std::error::Error>> {
         }
 
         Commands::AgentsMd { output } => {
-            let mem = HDF5Memory::open_read_only(&cli.path)?;
+            let mem = HDF5Memory::open_read_only(&path)?;
             let md = mem.generate_agents_md();
             match output {
                 Some(p) => {
@@ -210,7 +314,7 @@ fn run(cli: Cli) -> Result<(), Box<dyn std::error::Error>> {
         }
 
         Commands::Export => {
-            let mem = HDF5Memory::open_read_only(&cli.path)?;
+            let mem = HDF5Memory::open_read_only(&path)?;
             for i in 0..mem.count() {
                 if let Some(chunk) = mem.get_chunk(i) {
                     let j = serde_json::json!({ "index": i, "chunk": chunk });
@@ -219,11 +323,44 @@ fn run(cli: Cli) -> Result<(), Box<dyn std::error::Error>> {
             }
         }
 
+        Commands::Keygen { .. } => unreachable!("handled before opening a store"),
+
+        Commands::Verify { public_key } => {
+            let text = if Path::new(&public_key).is_file() {
+                std::fs::read_to_string(&public_key)?
+            } else {
+                public_key
+            };
+            let bytes = signing::from_hex::<32>(&text)
+                .ok_or("--public-key must be 64 hex characters or a file holding them")?;
+            let trusted = VerifyingKey::from_bytes(&bytes)?;
+            let r = HDF5Memory::verify(&path, &trusted)?;
+            let j = serde_json::json!({
+                "valid": r.is_valid(),
+                "signed": r.signed,
+                "key_matches": r.key_matches,
+                "signature_valid": r.signature_valid,
+                "records_match": r.records_match,
+                "settings_match": r.settings_match,
+                "sessions_match": r.sessions_match,
+                "graph_match": r.graph_match,
+                "changed_records": r.changed_records,
+                "record_count": r.record_count,
+                "signed_record_count": r.signed_record_count,
+                "signed_by": r.public_key.map(|k| signing::to_hex(&k)),
+                "wal_entries_unsigned": r.wal_entries_unsigned,
+            });
+            println!("{}", serde_json::to_string_pretty(&j)?);
+            if !r.is_valid() {
+                std::process::exit(2);
+            }
+        }
+
         Commands::Snapshot { dest } => {
-            let _result = clawhdf5_agent::storage::snapshot_file(&cli.path, &dest)?;
+            let _result = clawhdf5_agent::storage::snapshot_file(&path, &dest)?;
             let j = serde_json::json!({
                 "status": "snapshot_created",
-                "source": cli.path.display().to_string(),
+                "source": path.display().to_string(),
                 "dest": dest.display().to_string(),
             });
             println!("{}", serde_json::to_string(&j)?);

crates/clawhdf5-derive/Cargo.toml

@@ -2,6 +2,7 @@
 name = "clawhdf5-derive"
 version = "2.7.0"
 edition = "2024"
+rust-version.workspace = true
 description = "Derive macros for rustyhdf5 HDF5 traits"
 license = "MIT"
 repository = "https://git.redclaw.dev/quantumclaw/clawhdf5"

crates/clawhdf5-filters/Cargo.toml

@@ -2,6 +2,7 @@
 name = "clawhdf5-filters"
 version = "2.7.0"
 edition = "2024"
+rust-version.workspace = true
 description = "Filter and compression pipeline for clawhdf5"
 license = "MIT"
 repository = "https://git.redclaw.dev/quantumclaw/clawhdf5"
@@ -25,8 +26,12 @@ name = "compression_bench"
 harness = false
 
 [features]
-default = ["fast-deflate"]
+# Pure-Rust zlib-rs by default; `fast-deflate` (zlib-ng, C) overrides it.
+default = ["zlib-rs"]
 fast-deflate = ["flate2/zlib-ng"]
 system-zlib = ["flate2/zlib-default"]
-zlib-rs = ["flate2/zlib-rs"]
+# `runtime_detection` gives zlib-rs `std`, which it needs to detect and use
+# SIMD at runtime. flate2 enables it by default, but we build flate2 with
+# default-features = false, and without it zlib-rs inflates 3.5x slower.
+zlib-rs = ["flate2/zlib-rs", "flate2/runtime_detection"]
 apple-compression = []

crates/clawhdf5-filters/README.md

@@ -8,16 +8,18 @@ Filter and compression pipeline for clawhdf5.
 ## Features
 
 - DEFLATE compression/decompression
-- Fast deflate via zlib-ng (`fast-deflate` feature)
+- Pure-Rust deflate via zlib-rs (default, `zlib-rs` feature)
+- zlib-ng instead, if you want it (`fast-deflate` feature; C, needs cmake)
 - Apple Compression framework support (`apple-compression` feature)
 
 ## Usage
 
 ```rust
-use clawhdf5_filters::{deflate_decode, deflate_encode};
+use clawhdf5_filters::{deflate_compress, deflate_decompress};
 
-let compressed = deflate_encode(&data, 6).unwrap();
-let decompressed = deflate_decode(&compressed).unwrap();
+let compressed = deflate_compress(&data, 6).unwrap();
+// The second argument bounds the output: the expected decompressed size.
+let decompressed = deflate_decompress(&compressed, data.len()).unwrap();
 ```
 
 ## License

crates/clawhdf5-filters/src/fast_deflate.rs

@@ -1,12 +1,13 @@
-//! Fast deflate backends: Apple Compression Framework and zlib-ng.
+//! Deflate backends: Apple Compression Framework, zlib-ng and zlib-rs.
 //!
 //! Backend selection priority (decompression & compression):
 //! 1. Apple Compression Framework (macOS only, `apple-compression` feature)
-//! 2. flate2 with zlib-ng backend (`fast-deflate` feature) or miniz_oxide (default)
+//! 2. flate2 with zlib-ng (`fast-deflate`), else zlib-rs (`zlib-rs`, the
+//!    default), else miniz_oxide
 //!
 //! The Apple Compression Framework uses hardware-accelerated zlib on Apple Silicon
-//! and is typically the fastest option on macOS. zlib-ng is the fastest portable
-//! option and what C HDF5 uses internally.
+//! and is typically the fastest option on macOS. zlib-rs is a pure-Rust port of
+//! zlib-ng; see `BENCHMARKS.md` for how the two compare.
 
 // ---------------------------------------------------------------------------
 // Apple Compression Framework FFI (macOS only)
@@ -243,65 +244,117 @@ mod apple {
 }
 
 // ---------------------------------------------------------------------------
-// Streaming decompression via flate2 (uses zlib-ng when fast-deflate enabled)
+// One-shot (de)compression via flate2 (whichever backend flate2 was built with)
+//
+// The whole input goes to the codec in one call, into an output buffer sized
+// up front. `flate2::read::ZlibDecoder` / `write::ZlibEncoder` stream through a
+// 32 KiB buffer instead, which cost zlib-rs up to 3.7x against zlib-ng on a
+// 1 MB chunk. clawhdf5-format's deflate filter does the same; see
+// `BENCHMARKS.md`, "Deflate backend".
 // ---------------------------------------------------------------------------
 
-/// Streaming decompress with pre-allocated output buffer.
-///
-/// When the output size is known (typical for HDF5 chunks), this avoids
-/// dynamic reallocation by writing directly into a pre-sized buffer.
+/// Decompress into a buffer pre-sized to `output_size`, the expected
+/// decompressed length (known for HDF5 chunks). Output longer than that is an
+/// error, as is a stream that ends early.
 pub(crate) fn flate2_decompress_preallocated(
     data: &[u8],
     output_size: usize,
 ) -> Result<Vec<u8>, String> {
-    use std::io::Read;
-    let mut decoder = flate2::read::ZlibDecoder::new(data);
-    let mut output = vec![0u8; output_size];
-    let mut total_read = 0;
-
-    loop {
-        match decoder.read(&mut output[total_read..]) {
-            Ok(0) => break,
-            Ok(n) => total_read += n,
-            Err(e) => return Err(e.to_string()),
-        }
-    }
-    output.truncate(total_read);
-    Ok(output)
+    inflate_bounded(data, output_size, output_size)
 }
 
 /// Absolute ceiling on decompressed output when the caller has no size hint,
 /// preventing unbounded allocation from a hostile/corrupted zlib stream.
 const MAX_DECOMPRESS_SIZE: usize = 256 * 1024 * 1024;
 
-/// Streaming decompress with dynamic sizing (when output size is unknown).
-///
-/// Bounded by [`MAX_DECOMPRESS_SIZE`] since there is no chunk-size hint to
-/// validate against here — an unbounded `read_to_end` would let a hostile
-/// zlib stream force arbitrarily large allocation (a "zlib bomb").
+/// Decompress with no size hint, bounded by [`MAX_DECOMPRESS_SIZE`] so a
+/// hostile zlib stream cannot force arbitrarily large allocation (a "zlib
+/// bomb").
 pub(crate) fn flate2_decompress_streaming(data: &[u8]) -> Result<Vec<u8>, String> {
-    use std::io::Read;
-    let decoder = flate2::read::ZlibDecoder::new(data);
-    let mut result = Vec::new();
-    decoder
-        .take(MAX_DECOMPRESS_SIZE as u64 + 1)
-        .read_to_end(&mut result)
-        .map_err(|e| e.to_string())?;
-    if result.len() > MAX_DECOMPRESS_SIZE {
-        return Err(format!(
+    let hint = data.len().saturating_mul(4).min(1 << 20);
+    inflate_bounded(data, hint, MAX_DECOMPRESS_SIZE).map_err(|e| {
+        if e.ends_with("exceeds size limit") {
+            format!(
                 "decompressed output exceeds {} MiB limit",
                 MAX_DECOMPRESS_SIZE / 1024 / 1024
-        ));
+            )
+        } else {
+            e
         }
-    Ok(result)
+    })
 }
 
-/// Compress data using flate2 (zlib-ng when fast-deflate enabled, else miniz_oxide).
+/// Inflate a zlib stream, starting from `size_hint` bytes of output and
+/// failing past `limit`.
+fn inflate_bounded(data: &[u8], size_hint: usize, limit: usize) -> Result<Vec<u8>, String> {
+    use flate2::{Decompress, FlushDecompress, Status};
+
+    // One byte of headroom past the limit distinguishes an over-size stream
+    // from one that legitimately ends exactly at the limit.
+    let max_capacity = limit.saturating_add(1);
+    let mut out = Vec::new();
+    out.try_reserve_exact(size_hint.clamp(1, max_capacity))
+        .map_err(|e| format!("deflate: cannot allocate output: {e}"))?;
+
+    let mut inflater = Decompress::new(true);
+    loop {
+        let (in_before, out_before) = (inflater.total_in(), inflater.total_out());
+        let status = inflater
+            .decompress_vec(
+                &data[in_before as usize..],
+                &mut out,
+                FlushDecompress::Finish,
+            )
+            .map_err(|e| format!("deflate: {e}"))?;
+        if out.len() > limit {
+            return Err("deflate: output exceeds size limit".into());
+        }
+        match status {
+            Status::StreamEnd => return Ok(out),
+            Status::Ok | Status::BufError if out.len() == out.capacity() => {
+                let grow = out.capacity().min(max_capacity - out.capacity()).max(1);
+                out.try_reserve_exact(grow)
+                    .map_err(|e| format!("deflate: cannot allocate output: {e}"))?;
+            }
+            Status::Ok | Status::BufError => {
+                if inflater.total_in() as usize >= data.len()
+                    || (inflater.total_in(), inflater.total_out()) == (in_before, out_before)
+                {
+                    return Err("deflate: truncated stream".into());
+                }
+            }
+        }
+    }
+}
+
+/// Compress data using flate2 (zlib-ng, zlib-rs or miniz_oxide; see module docs).
 pub(crate) fn flate2_compress(data: &[u8], level: u32) -> Result<Vec<u8>, String> {
-    use std::io::Write;
-    let mut encoder = flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::new(level));
-    encoder.write_all(data).map_err(|e| e.to_string())?;
-    encoder.finish().map_err(|e| e.to_string())
+    use flate2::{Compress, Compression, FlushCompress, Status};
+
+    // zlib's compressBound, plus the zlib header and trailer.
+    let bound = data.len() + (data.len() >> 12) + (data.len() >> 14) + (data.len() >> 25) + 13 + 6;
+    let mut out = Vec::new();
+    out.try_reserve_exact(bound)
+        .map_err(|e| format!("deflate: cannot allocate output: {e}"))?;
+
+    let mut deflater = Compress::new(Compression::new(level), true);
+    loop {
+        let (in_before, out_before) = (deflater.total_in(), deflater.total_out());
+        let status = deflater
+            .compress_vec(&data[in_before as usize..], &mut out, FlushCompress::Finish)
+            .map_err(|e| format!("deflate: {e}"))?;
+        match status {
+            Status::StreamEnd => return Ok(out),
+            Status::Ok | Status::BufError if out.len() == out.capacity() => out
+                .try_reserve(out.capacity().max(4096))
+                .map_err(|e| format!("deflate: cannot allocate output: {e}"))?,
+            Status::Ok | Status::BufError => {
+                if (deflater.total_in(), deflater.total_out()) == (in_before, out_before) {
+                    return Err("deflate: encoder made no progress".into());
+                }
+            }
+        }
+    }
 }
 
 // ---------------------------------------------------------------------------
@@ -312,7 +365,7 @@ pub(crate) fn flate2_compress(data: &[u8], level: u32) -> Result<Vec<u8>, String
 ///
 /// Selection order:
 /// 1. Apple Compression Framework (macOS + `apple-compression` feature)
-/// 2. flate2 (zlib-ng with `fast-deflate`, otherwise miniz_oxide)
+/// 2. flate2 (zlib-ng with `fast-deflate`, else zlib-rs, else miniz_oxide)
 ///
 /// When `output_hint` > 0, pre-allocates the output buffer for zero-copy
 /// decompression (avoids reallocation).
@@ -344,7 +397,7 @@ pub fn decompress(data: &[u8], output_hint: usize) -> Result<Vec<u8>, String> {
 ///
 /// Selection order:
 /// 1. Apple Compression Framework (macOS + `apple-compression` feature)
-/// 2. flate2 (zlib-ng with `fast-deflate`, otherwise miniz_oxide)
+/// 2. flate2 (zlib-ng with `fast-deflate`, else zlib-rs, else miniz_oxide)
 pub fn compress(data: &[u8], level: u32) -> Result<Vec<u8>, String> {
     #[cfg(all(target_os = "macos", feature = "apple-compression"))]
     {
@@ -377,9 +430,19 @@ pub fn active_backend() -> &'static str {
     {
         "zlib-ng"
     }
+    // flate2 prefers a C zlib over zlib-rs when both are enabled.
+    #[cfg(all(
+        not(all(target_os = "macos", feature = "apple-compression")),
+        not(feature = "fast-deflate"),
+        feature = "zlib-rs"
+    ))]
+    {
+        "zlib-rs"
+    }
     #[cfg(not(any(
         all(target_os = "macos", feature = "apple-compression"),
-        feature = "fast-deflate"
+        feature = "fast-deflate",
+        feature = "zlib-rs"
     )))]
     {
         "miniz_oxide"
@@ -436,7 +499,7 @@ mod tests {
     fn backend_name_is_set() {
         let name = active_backend();
         assert!(
-            ["miniz_oxide", "zlib-ng", "apple-compression"].contains(&name),
+            ["miniz_oxide", "zlib-rs", "zlib-ng", "apple-compression"].contains(&name),
             "unexpected backend: {name}"
         );
     }

crates/clawhdf5-filters/src/lib.rs

@@ -2,12 +2,14 @@
 //!
 //! Provides deflate (zlib) decompression/compression with multiple backend options:
 //!
-//! - **Default**: `miniz_oxide` (pure Rust, no C dependencies)
-//! - **`fast-deflate` feature**: `zlib-ng` via flate2 (~2-3x faster, matches C HDF5)
+//! - **Default (`zlib-rs` feature)**: `zlib-rs` via flate2 (pure Rust, no C
+//!   dependencies)
+//! - **`fast-deflate` feature**: `zlib-ng` via flate2 (C, built with cmake)
 //! - **`apple-compression` feature**: Apple Compression Framework on macOS
 //!   (hardware-accelerated on Apple Silicon)
+//! - With none of the above: `miniz_oxide` (pure Rust, slower)
 //!
-//! Backend priority: apple-compression > zlib-ng > miniz_oxide.
+//! Backend priority: apple-compression > zlib-ng > zlib-rs > miniz_oxide.
 
 pub mod fast_deflate;
 
@@ -115,7 +117,7 @@ mod tests {
     fn backend_reports_name() {
         let name = deflate_backend();
         assert!(
-            ["miniz_oxide", "zlib-ng", "apple-compression"].contains(&name),
+            ["miniz_oxide", "zlib-rs", "zlib-ng", "apple-compression"].contains(&name),
             "unexpected backend: {name}"
         );
     }

crates/clawhdf5-format/Cargo.toml

@@ -2,6 +2,7 @@
 name = "clawhdf5-format"
 version = "2.7.0"
 edition = "2024"
+rust-version.workspace = true
 description = "Pure-Rust HDF5 binary format parsing and writing — no C dependencies"
 license = "MIT"
 repository = "https://git.redclaw.dev/quantumclaw/clawhdf5"
@@ -23,6 +24,7 @@ libaec-sys = { path = "../libaec-sys", version = "0.1", optional = true }
 pco = { version = "1.0", optional = true }
 
 [dev-dependencies]
+half = { workspace = true }
 serde_json = "1"
 criterion = { workspace = true }
 clawhdf5-derive = { path = "../clawhdf5-derive", version = "2.7.0" }
@@ -32,7 +34,10 @@ name = "bench"
 harness = false
 
 [features]
-default = ["std", "checksum", "deflate", "provenance", "fast-deflate", "system-zlib-decompress"]
+# Deflate backend: `zlib-rs` (pure Rust) by default. `fast-deflate` selects
+# zlib-ng instead (C, built with cmake); flate2 prefers a C zlib whenever one
+# is enabled, so turning it on anywhere in the build overrides the default.
+default = ["std", "checksum", "deflate", "provenance", "zlib-rs", "system-zlib-decompress"]
 std = []
 checksum = []
 deflate = ["flate2"]
@@ -42,7 +47,10 @@ fast-checksum = ["crc32fast"]
 fast-deflate = ["flate2/zlib-ng"]
 system-zlib = ["flate2/zlib-default"]
 system-zlib-decompress = []
-zlib-rs = ["flate2/zlib-rs"]
+# `runtime_detection` gives zlib-rs `std`, which it needs to detect and use
+# SIMD at runtime. flate2 enables it by default, but we build flate2 with
+# default-features = false, and without it zlib-rs inflates 3.5x slower.
+zlib-rs = ["flate2/zlib-rs", "flate2/runtime_detection"]
 lz4 = ["lz4_flex"]
 zstd = ["dep:zstd"]
 blake3_hash = ["blake3"]

crates/clawhdf5-format/src/chunk_cache.rs

@@ -223,13 +223,32 @@ pub const DEFAULT_CACHE_BYTES: usize = 16 * 1024 * 1024; // 16 MiB
 /// coordinate map and reduces collision chains compared to power-of-two sizes.
 pub const DEFAULT_MAX_SLOTS: usize = 521;
 
+/// Most datasets whose chunk index a [`ChunkCache`] keeps at once.
+pub const MAX_INDEXED_DATASETS: usize = 64;
+
+/// Most chunk-index entries, summed over all datasets, a [`ChunkCache`] keeps.
+/// Least-recently-used datasets' indexes are dropped past this (the dataset
+/// being read is always kept), so a file with many or huge chunked datasets
+/// cannot grow the cache without bound.
+pub const MAX_INDEXED_CHUNKS: usize = 1 << 20;
+
+/// The dataset key the address-less (legacy) methods use when
+/// [`ChunkCache::ensure_dataset`] has not been called.
+#[cfg(feature = "std")]
+const UNBOUND_DATASET: u64 = u64::MAX;
+
 // ---------------------------------------------------------------------------
 // LRU entry
 // ---------------------------------------------------------------------------
 
+/// Decompressed chunks are keyed by dataset *and* coordinate: every chunked
+/// dataset has a chunk at (0, 0, ...), so the coordinate alone is ambiguous.
+#[cfg(feature = "std")]
+type SlotKey = (u64, ChunkCoord);
+
 #[cfg(feature = "std")]
 struct CachedChunk {
-    coord: ChunkCoord,
+    key: SlotKey,
     /// Shared so a cache hit is a refcount bump, not a copy of the whole
     /// (potentially large) decompressed chunk.
     data: Arc<CacheAlignedBuffer>,
@@ -237,21 +256,48 @@ struct CachedChunk {
     last_access: u64,
 }
 
+/// Per-dataset index state.
+#[cfg(feature = "std")]
+#[derive(Default)]
+struct DatasetEntry {
+    /// Chunk coordinate -> ChunkInfo (offset + size in file).
+    index: Option<Arc<HashMap<ChunkCoord, ChunkInfo>>>,
+    /// Pre-built chunk index for O(1) coordinate lookups.
+    chunk_index: Option<Arc<ChunkIndex>>,
+    /// Pre-computed chunk layout for fast assembly.
+    chunk_layout: Option<Arc<ChunkLayout>>,
+    /// Tick of the last use, for dropping the least recently used dataset.
+    last_used: u64,
+}
+
+#[cfg(feature = "std")]
+impl DatasetEntry {
+    fn weight(&self) -> usize {
+        self.index.as_ref().map_or(0, |m| m.len())
+            + self.chunk_index.as_ref().map_or(0, |c| c.num_chunks())
+    }
+}
+
 // ---------------------------------------------------------------------------
 // ChunkCache
 // ---------------------------------------------------------------------------
 
-/// A per-dataset chunk cache with hash-based index and LRU eviction.
+/// A per-file chunk cache: chunk indexes per dataset, plus an LRU of
+/// decompressed chunks, all keyed by dataset.
 ///
-/// # Usage
+/// A dataset is identified by the address of its chunk index (B-tree, fixed
+/// or extensible array, ...), which is unique within a file. Every method
+/// that takes an `addr` works on that dataset only, so threads reading
+/// different datasets through one shared cache never see each other's
+/// chunks. The address-less methods (`has_index`, `populate_index`,
+/// `get_decompressed`, ...) act on the dataset last bound with
+/// [`Self::ensure_dataset`]; that binding is shared state, so concurrent
+/// readers must use the `*_in` / `*_for` methods instead (the chunked
+/// readers in [`crate::chunked_read`] do).
 ///
-/// ```ignore
-/// let cache = ChunkCache::new();
-/// // Pass &cache to read_chunked_data — it will populate the index lazily.
-/// ```
-///
-/// The cache is wrapped in `Mutex` internally so it can be mutated through
-/// shared references (thread-safe).
+/// Memory is bounded: decompressed data by `max_bytes`/`max_slots` across
+/// all datasets, indexes by [`MAX_INDEXED_DATASETS`] and
+/// [`MAX_INDEXED_CHUNKS`].
 ///
 /// Only available with the `std` feature because it requires `std::sync::Mutex`.
 #[cfg(feature = "std")]
@@ -261,26 +307,20 @@ pub struct ChunkCache {
 
 #[cfg(feature = "std")]
 struct CacheInner {
-    /// Hash index: chunk coordinate -> ChunkInfo (offset + size in file).
-    /// Populated once per dataset on first access.
-    index: Option<HashMap<ChunkCoord, ChunkInfo>>,
+    /// Per-dataset chunk indexes, keyed by chunk-index address.
+    datasets: HashMap<u64, DatasetEntry>,
 
-    /// Address of the dataset (its chunk-index base address) that the cached
-    /// index, chunk index, layout, and decompressed slots currently belong to.
-    /// The cache is shared per file across datasets, so every cached-read entry
-    /// checks this and resets the per-dataset state when the dataset changes —
-    /// otherwise one dataset's chunk index (with its own rank) would be reused
-    /// for another, corrupting reads.
-    index_addr: Option<u64>,
+    /// Dataset the address-less methods act on (see `ensure_dataset`).
+    current: Option<u64>,
 
     /// LRU cache of decompressed chunk data.
     slots: Vec<CachedChunk>,
 
-    /// Coordinate -> index into `slots`, for O(1) lookup instead of a linear
+    /// Key -> index into `slots`, for O(1) lookup instead of a linear
     /// scan. Kept in sync with `slots` on every insert/evict/clear — in
     /// particular, `slots.swap_remove(i)` moves the last element into slot
     /// `i`, so the moved element's index entry must be updated too.
-    slot_index: HashMap<ChunkCoord, usize>,
+    slot_index: HashMap<SlotKey, usize>,
 
     /// Current total bytes of cached decompressed data.
     current_bytes: usize,
@@ -294,17 +334,145 @@ struct CacheInner {
     /// Monotonic counter for LRU ordering.
     tick: u64,
 
-    /// Last accessed chunk coordinate (for sequential detection).
-    last_coord: Option<ChunkCoord>,
+    /// Last accessed chunk (for sequential detection).
+    last_coord: Option<SlotKey>,
 
     /// Access pattern statistics.
     stats: AccessStats,
+}
 
-    /// Pre-built chunk index for O(1) coordinate lookups.
-    chunk_index: Option<ChunkIndex>,
+#[cfg(feature = "std")]
+impl CacheInner {
+    fn current(&self) -> u64 {
+        self.current.unwrap_or(UNBOUND_DATASET)
+    }
 
-    /// Pre-computed chunk layout for fast assembly.
-    chunk_layout: Option<ChunkLayout>,
+    fn touch(&mut self, addr: u64) -> &mut DatasetEntry {
+        self.tick += 1;
+        let tick = self.tick;
+        let entry = self.datasets.entry(addr).or_default();
+        entry.last_used = tick;
+        entry
+    }
+
+    fn entry(&self, addr: u64) -> Option<&DatasetEntry> {
+        self.datasets.get(&addr)
+    }
+
+    /// Drop least-recently-used datasets' indexes (never `keep`'s) until the
+    /// dataset and chunk-entry budgets hold.
+    fn trim_datasets(&mut self, keep: u64) {
+        loop {
+            let total: usize = self.datasets.values().map(DatasetEntry::weight).sum();
+            if self.datasets.len() <= MAX_INDEXED_DATASETS && total <= MAX_INDEXED_CHUNKS {
+                return;
+            }
+            let victim = self
+                .datasets
+                .iter()
+                .filter(|(a, _)| **a != keep)
+                .min_by_key(|(_, e)| e.last_used)
+                .map(|(a, _)| *a);
+            match victim {
+                Some(a) => {
+                    self.datasets.remove(&a);
+                }
+                None => return,
+            }
+        }
+    }
+
+    fn get_decompressed(&mut self, addr: u64, coord: &[u64]) -> Option<Arc<CacheAlignedBuffer>> {
+        self.tick += 1;
+        let tick = self.tick;
+
+        // Track sequential vs random access
+        let is_sequential = self.last_coord.as_ref().is_some_and(|(prev_addr, prev)| {
+            // Sequential if exactly one dimension changed
+            let changes: usize = prev
+                .iter()
+                .zip(coord.iter())
+                .filter(|(a, b)| a != b)
+                .count();
+            *prev_addr == addr && changes <= 1
+        });
+        if is_sequential {
+            self.stats.sequential_count += 1;
+        } else if self.last_coord.is_some() {
+            self.stats.random_count += 1;
+        }
+        let key: SlotKey = (addr, coord.to_vec());
+        let found = if let Some(&idx) = self.slot_index.get(&key) {
+            self.slots[idx].last_access = tick;
+            Some(Arc::clone(&self.slots[idx].data))
+        } else {
+            None
+        };
+        self.last_coord = Some(key);
+        if let Some(ref data) = found {
+            self.stats.hits += 1;
+            self.stats.bytes_read += data.len() as u64;
+        } else {
+            self.stats.misses += 1;
+        }
+        found
+    }
+
+    fn put_decompressed(
+        &mut self,
+        key: SlotKey,
+        data: Arc<CacheAlignedBuffer>,
+    ) -> Arc<CacheAlignedBuffer> {
+        let data_len = data.len();
+
+        // Don't cache if single chunk exceeds budget — still return the data
+        // to the caller, just don't retain it.
+        if data_len > self.max_bytes {
+            return data;
+        }
+
+        // Check if already present
+        self.tick += 1;
+        let tick = self.tick;
+        if let Some(&idx) = self.slot_index.get(&key) {
+            self.slots[idx].last_access = tick;
+            return Arc::clone(&self.slots[idx].data); // already cached
+        }
+
+        // Evict until we have room
+        while self.slots.len() >= self.max_slots
+            || (self.current_bytes + data_len > self.max_bytes && !self.slots.is_empty())
+        {
+            // Find LRU slot
+            let lru_idx = self
+                .slots
+                .iter()
+                .enumerate()
+                .min_by_key(|(_, s)| s.last_access)
+                .map(|(i, _)| i)
+                .unwrap();
+            let removed = self.slots.swap_remove(lru_idx);
+            self.slot_index.remove(&removed.key);
+            // swap_remove moved the former last element into `lru_idx` (unless
+            // it *was* the last element) — fix up that element's index entry.
+            if lru_idx < self.slots.len() {
+                let moved_key = self.slots[lru_idx].key.clone();
+                self.slot_index.insert(moved_key, lru_idx);
+            }
+            self.current_bytes -= removed.data.len();
+            self.stats.evictions += 1;
+        }
+
+        self.current_bytes += data_len;
+        let new_idx = self.slots.len();
+        self.slot_index.insert(key.clone(), new_idx);
+        self.slots.push(CachedChunk {
+            key,
+            data: Arc::clone(&data),
+            last_access: tick,
+        });
+        data
+    }
 }
 
 /// Access pattern statistics tracked by the chunk cache.
@@ -356,8 +524,8 @@ impl ChunkCache {
     pub fn with_capacity(max_bytes: usize, max_slots: usize) -> Self {
         Self {
             inner: std::sync::Mutex::new(CacheInner {
-                index: None,
-                index_addr: None,
+                datasets: HashMap::new(),
+                current: None,
                 slots: Vec::with_capacity(max_slots.min(64)),
                 slot_index: HashMap::with_capacity(max_slots.min(64)),
                 current_bytes: 0,
@@ -366,340 +534,331 @@ impl ChunkCache {
                 tick: 0,
                 last_coord: None,
                 stats: AccessStats::default(),
-                chunk_index: None,
-                chunk_layout: None,
             }),
         }
     }
 
-    // ----- Index operations -----
+    fn lock(&self) -> std::sync::MutexGuard<'_, CacheInner> {
+        self.inner.lock().unwrap_or_else(|e| e.into_inner())
+    }
 
     /// The most decompressed bytes this cache will hold.
     pub fn max_bytes(&self) -> usize {
-        self.inner.lock().map(|g| g.max_bytes).unwrap_or(0)
+        self.lock().max_bytes
     }
 
-    /// Bind the cache to the dataset at chunk-index address `addr`.
+    // ----- Dataset-keyed operations (safe to use concurrently) -----
+
+    /// The chunk list of the dataset whose chunk index is at `addr`.
     ///
-    /// The cache is shared per file across all of its datasets. If the cache
-    /// currently holds state for a different dataset, all per-dataset state
-    /// (chunk index, chunk-index map, layout, and decompressed slots) is
-    /// dropped so the next access rebuilds it for this dataset. Reading the
-    /// same dataset again is a no-op, preserving the cache's benefit for
-    /// repeated/sequential access. Returns `true` if a reset occurred.
+    /// On the first call for a dataset, `build` scans its chunk index; the
+    /// result is kept (offsets truncated to `rank` for the lookup key), so
+    /// later calls skip the scan. `build` runs without the cache lock held;
+    /// if two threads race to build the same dataset's index, the first
+    /// stored one wins and both return equivalent lists.
+    pub fn chunks_for<E>(
+        &self,
+        addr: u64,
+        rank: usize,
+        build: impl FnOnce() -> Result<Vec<ChunkInfo>, E>,
+    ) -> Result<Vec<ChunkInfo>, E> {
+        Ok(self
+            .index_for(addr, rank, build)?
+            .values()
+            .cloned()
+            .collect())
+    }
+
+    fn index_for<E>(
+        &self,
+        addr: u64,
+        rank: usize,
+        build: impl FnOnce() -> Result<Vec<ChunkInfo>, E>,
+    ) -> Result<Arc<HashMap<ChunkCoord, ChunkInfo>>, E> {
+        if let Some(index) = self.lock().touch(addr).index.clone() {
+            return Ok(index);
+        }
+        let chunks = build()?;
+        let map: HashMap<ChunkCoord, ChunkInfo> = chunks
+            .into_iter()
+            .map(|ci| (ci.offsets.iter().take(rank).copied().collect(), ci))
+            .collect();
+        let mut inner = self.lock();
+        let entry = inner.touch(addr);
+        let index = Arc::clone(entry.index.get_or_insert_with(|| Arc::new(map)));
+        inner.trim_datasets(addr);
+        Ok(index)
+    }
+
+    /// The pre-computed assembly layout of the dataset at `addr`, building
+    /// its chunk index (via `build`, as in [`Self::chunks_for`]) and layout on
+    /// first use.
+    pub fn chunk_layout_for<E>(
+        &self,
+        addr: u64,
+        rank: usize,
+        build: impl FnOnce() -> Result<Vec<ChunkInfo>, E>,
+        ds_dims: &[usize],
+        chunk_dims: &[usize],
+        elem_size: usize,
+    ) -> Result<Arc<ChunkLayout>, E> {
+        let (layout, chunk_index) = {
+            let mut inner = self.lock();
+            let entry = inner.touch(addr);
+            (entry.chunk_layout.clone(), entry.chunk_index.clone())
+        };
+        if let Some(layout) = layout {
+            return Ok(layout);
+        }
+        let chunk_index = match chunk_index {
+            Some(ci) => ci,
+            None => {
+                let index = self.index_for(addr, rank, build)?;
+                let chunks: Vec<ChunkInfo> = index.values().cloned().collect();
+                Arc::new(ChunkIndex::build(&chunks, rank))
+            }
+        };
+        let layout = ChunkLayout::build(&chunk_index, ds_dims, chunk_dims, elem_size);
+        let mut inner = self.lock();
+        let entry = inner.touch(addr);
+        entry.chunk_index.get_or_insert(chunk_index);
+        let layout = Arc::clone(entry.chunk_layout.get_or_insert_with(|| Arc::new(layout)));
+        inner.trim_datasets(addr);
+        Ok(layout)
+    }
+
+    /// Cached decompressed chunk at `coord` of the dataset at `addr`.
+    ///
+    /// O(1) lookup; the clone is an `Arc` refcount bump, not a copy of the
+    /// underlying decompressed data.
+    pub fn get_decompressed_in(&self, addr: u64, coord: &[u64]) -> Option<Arc<CacheAlignedBuffer>> {
+        self.lock().get_decompressed(addr, coord)
+    }
+
+    /// Cache decompressed chunk data for `coord` of the dataset at `addr`.
+    /// Returns the `Arc`-shared buffer now cached (or already cached).
+    pub fn put_decompressed_in(
+        &self,
+        addr: u64,
+        coord: ChunkCoord,
+        data: Vec<u8>,
+    ) -> Arc<CacheAlignedBuffer> {
+        self.put_decompressed_aligned_in(addr, coord, CacheAlignedBuffer::from_vec(data))
+    }
+
+    /// [`Self::put_decompressed_in`] for an already-aligned buffer.
+    pub fn put_decompressed_aligned_in(
+        &self,
+        addr: u64,
+        coord: ChunkCoord,
+        data: CacheAlignedBuffer,
+    ) -> Arc<CacheAlignedBuffer> {
+        let data = Arc::new(data);
+        self.lock().put_decompressed((addr, coord), data)
+    }
+
+    /// Record that the given chunk coordinates of the dataset at `addr` are
+    /// predicted to be accessed soon (bookkeeping only).
+    ///
+    /// This does **not** prefetch or pre-decompress anything — it only
+    /// checks whether each coordinate is already in the chunk index and
+    /// updates access-pattern stats accordingly.
+    pub fn prefetch_hint_in(&self, addr: u64, next_coords: &[ChunkCoord]) {
+        let mut inner = self.lock();
+        let Some(index) = inner.entry(addr).and_then(|e| e.index.clone()) else {
+            return;
+        };
+        let known = next_coords
+            .iter()
+            .filter(|c| index.contains_key(*c))
+            .count();
+        inner.stats.sequential_count += known as u64;
+    }
+
+    // ----- Address-less operations on the bound dataset -----
+
+    /// Bind the address-less methods to the dataset at chunk-index address
+    /// `addr`. Returns `true` if this changed the bound dataset.
+    ///
+    /// Each dataset's state is kept separately, so switching loses nothing
+    /// and never exposes one dataset's index or chunks to another. The
+    /// binding itself is shared, though: concurrent readers should use the
+    /// `addr`-taking methods rather than bind and then call these.
     pub fn ensure_dataset(&self, addr: u64) -> bool {
-        let mut inner = self.inner.lock().unwrap_or_else(|e| e.into_inner());
-        if inner.index_addr == Some(addr) {
-            return false;
-        }
-        inner.index = None;
-        inner.chunk_index = None;
-        inner.chunk_layout = None;
-        inner.slots.clear();
-        inner.slot_index.clear();
-        inner.current_bytes = 0;
-        inner.last_coord = None;
-        inner.index_addr = Some(addr);
-        true
+        let mut inner = self.lock();
+        let changed = inner.current != Some(addr);
+        inner.current = Some(addr);
+        changed
     }
 
-    /// Returns `true` if the chunk index has been built.
+    /// Returns `true` if the bound dataset's chunk index has been built.
     pub fn has_index(&self) -> bool {
-        self.inner
-            .lock()
-            .unwrap_or_else(|e| e.into_inner())
-            .index
-            .is_some()
+        let inner = self.lock();
+        inner
+            .entry(inner.current())
+            .is_some_and(|e| e.index.is_some())
     }
 
-    /// Build the chunk index from a pre-collected list of `ChunkInfo`.
+    /// Build the bound dataset's chunk index from a pre-collected list of
+    /// `ChunkInfo`.
     ///
     /// The `rank` parameter is used to truncate offsets to spatial dims only
     /// (B-tree v1 stores rank+1 offsets).
     pub fn populate_index(&self, chunks: &[ChunkInfo], rank: usize) {
-        let mut inner = self.inner.lock().unwrap_or_else(|e| e.into_inner());
-        if inner.index.is_some() {
-            return; // already populated
-        }
-        let mut map = HashMap::with_capacity(chunks.len());
-
-        for ci in chunks {
-            let coord: ChunkCoord = ci.offsets.iter().take(rank).copied().collect();
-            map.insert(coord, ci.clone());
-        }
-        inner.index = Some(map);
+        let addr = self.lock().current();
+        let _ = self.index_for::<core::convert::Infallible>(addr, rank, || Ok(chunks.to_vec()));
     }
 
-    /// Look up a chunk by its spatial coordinate in the index.
+    /// Look up a chunk by its spatial coordinate in the bound dataset's index.
     pub fn lookup_index(&self, coord: &[u64]) -> Option<ChunkInfo> {
-        let inner = self.inner.lock().unwrap_or_else(|e| e.into_inner());
-        inner.index.as_ref()?.get(coord).cloned()
+        let inner = self.lock();
+        inner
+            .entry(inner.current())?
+            .index
+            .as_ref()?
+            .get(coord)
+            .cloned()
     }
 
-    /// Return all indexed chunks as a `Vec<ChunkInfo>` (order unspecified).
+    /// Return all of the bound dataset's indexed chunks (order unspecified).
     pub fn all_indexed_chunks(&self) -> Option<Vec<ChunkInfo>> {
-        let inner = self.inner.lock().unwrap_or_else(|e| e.into_inner());
-        inner.index.as_ref().map(|m| m.values().cloned().collect())
+        let inner = self.lock();
+        let index = inner.entry(inner.current())?.index.as_ref()?;
+        Some(index.values().cloned().collect())
     }
 
-    // ----- Chunk index (pre-built coordinate → ChunkInfo map) -----
-
-    /// Returns `true` if the chunk B-tree index has been built.
+    /// Returns `true` if the bound dataset's `ChunkIndex` has been built.
     pub fn has_chunk_index(&self) -> bool {
-        self.inner
-            .lock()
-            .unwrap_or_else(|e| e.into_inner())
-            .chunk_index
-            .is_some()
+        let inner = self.lock();
+        inner
+            .entry(inner.current())
+            .is_some_and(|e| e.chunk_index.is_some())
     }
 
-    /// Build and store the chunk B-tree index from a pre-collected list of `ChunkInfo`.
+    /// Build and store the bound dataset's `ChunkIndex`.
     pub fn populate_chunk_index(&self, chunks: &[ChunkInfo], rank: usize) {
-        let mut inner = self.inner.lock().unwrap_or_else(|e| e.into_inner());
-        if inner.chunk_index.is_some() {
-            return;
-        }
-        inner.chunk_index = Some(ChunkIndex::build(chunks, rank));
+        let built = Arc::new(ChunkIndex::build(chunks, rank));
+        let mut inner = self.lock();
+        let addr = inner.current();
+        inner.touch(addr).chunk_index.get_or_insert(built);
+        inner.trim_datasets(addr);
     }
 
-    // ----- Chunk layout (pre-computed assembly plan) -----
-
-    /// Returns `true` if the chunk layout has been computed.
+    /// Returns `true` if the bound dataset's chunk layout has been computed.
     pub fn has_chunk_layout(&self) -> bool {
-        self.inner
-            .lock()
-            .unwrap_or_else(|e| e.into_inner())
-            .chunk_layout
-            .is_some()
+        let inner = self.lock();
+        inner
+            .entry(inner.current())
+            .is_some_and(|e| e.chunk_layout.is_some())
     }
 
-    /// Build and store the pre-computed chunk layout for fast assembly.
+    /// Build and store the bound dataset's chunk layout (needs its
+    /// `ChunkIndex`; does nothing without one).
     pub fn populate_chunk_layout(&self, ds_dims: &[usize], chunk_dims: &[usize], elem_size: usize) {
-        let mut inner = self.inner.lock().unwrap_or_else(|e| e.into_inner());
-        if inner.chunk_layout.is_some() {
+        let mut inner = self.lock();
+        let addr = inner.current();
+        let entry = inner.touch(addr);
+        if entry.chunk_layout.is_some() {
             return;
         }
-        if let Some(ref idx) = inner.chunk_index {
-            inner.chunk_layout = Some(ChunkLayout::build(idx, ds_dims, chunk_dims, elem_size));
+        if let Some(idx) = entry.chunk_index.clone() {
+            entry.chunk_layout = Some(Arc::new(ChunkLayout::build(
+                &idx, ds_dims, chunk_dims, elem_size,
+            )));
         }
     }
 
-    /// Execute a function with a reference to the chunk layout.
-    ///
-    /// Returns `None` if the layout hasn't been computed yet.
+    /// Execute a function with a reference to the bound dataset's chunk
+    /// layout. Returns `None` if the layout hasn't been computed yet.
     pub fn with_chunk_layout<F, R>(&self, f: F) -> Option<R>
     where
         F: FnOnce(&ChunkLayout) -> R,
     {
-        let inner = self.inner.lock().unwrap_or_else(|e| e.into_inner());
-        inner.chunk_layout.as_ref().map(f)
+        let layout = {
+            let inner = self.lock();
+            inner.entry(inner.current())?.chunk_layout.clone()?
+        };
+        Some(f(&layout))
     }
 
-    // ----- Decompressed data cache (LRU) -----
-
-    /// Try to get cached decompressed data for a chunk coordinate.
+    /// Try to get cached decompressed data for a chunk of the bound dataset.
     ///
-    /// O(1) lookup. Returns an owned copy for API compatibility with callers
-    /// that need a `Vec<u8>`; prefer [`Self::get_decompressed_aligned`] when
-    /// an `Arc`-shared buffer works for the caller, since that avoids the
-    /// copy entirely.
+    /// Returns an owned copy; prefer [`Self::get_decompressed_aligned`] when
+    /// an `Arc`-shared buffer works for the caller.
     pub fn get_decompressed(&self, coord: &[u64]) -> Option<Vec<u8>> {
         self.get_decompressed_aligned(coord)
             .map(|arc| arc.as_slice().to_vec())
     }
 
-    /// Try to get a reference-counted clone of the aligned buffer for a chunk.
-    ///
-    /// O(1) index lookup; the clone is an `Arc` refcount bump, not a copy of
-    /// the underlying decompressed data.
+    /// Reference-counted cached buffer for a chunk of the bound dataset.
     pub fn get_decompressed_aligned(&self, coord: &[u64]) -> Option<Arc<CacheAlignedBuffer>> {
-        let mut inner = self.inner.lock().unwrap_or_else(|e| e.into_inner());
-        inner.tick += 1;
-        let tick = inner.tick;
-
-        // Track sequential vs random access
-        let is_sequential = inner.last_coord.as_ref().is_some_and(|prev| {
-            // Sequential if exactly one dimension changed
-            let changes: usize = prev
-                .iter()
-                .zip(coord.iter())
-                .filter(|(a, b)| a != b)
-                .count();
-            changes <= 1
-        });
-        if is_sequential {
-            inner.stats.sequential_count += 1;
-        } else if inner.last_coord.is_some() {
-            inner.stats.random_count += 1;
-        }
-        inner.last_coord = Some(coord.to_vec());
-
-        let found = if let Some(&idx) = inner.slot_index.get(coord) {
-            inner.slots[idx].last_access = tick;
-            Some(Arc::clone(&inner.slots[idx].data))
-        } else {
-            None
-        };
-        if let Some(ref data) = found {
-            inner.stats.hits += 1;
-            inner.stats.bytes_read += data.len() as u64;
-        } else {
-            inner.stats.misses += 1;
-        }
-        found
+        let mut inner = self.lock();
+        let addr = inner.current();
+        inner.get_decompressed(addr, coord)
     }
 
-    /// Insert decompressed chunk data into the LRU cache.
-    ///
-    /// The data is stored in a [`CacheAlignedBuffer`] so subsequent reads
-    /// return cache-line-aligned memory. Returns the `Arc`-shared buffer that
-    /// is now cached (or already was), so the caller can reuse it directly
-    /// instead of holding a separate copy of the same data.
+    /// Insert decompressed chunk data for the bound dataset into the LRU
+    /// cache, returning the `Arc`-shared buffer now cached.
     pub fn put_decompressed(&self, coord: ChunkCoord, data: Vec<u8>) -> Arc<CacheAlignedBuffer> {
-        let aligned = CacheAlignedBuffer::from_vec(data);
-        self.put_decompressed_aligned(coord, aligned)
+        self.put_decompressed_aligned(coord, CacheAlignedBuffer::from_vec(data))
     }
 
-    /// Insert an already-aligned buffer into the LRU cache.
-    ///
-    /// Returns the `Arc`-shared buffer now held by the cache (the one just
-    /// inserted, or the existing cached copy if `coord` was already present).
+    /// Insert an already-aligned buffer for the bound dataset.
     pub fn put_decompressed_aligned(
         &self,
         coord: ChunkCoord,
         data: CacheAlignedBuffer,
     ) -> Arc<CacheAlignedBuffer> {
         let data = Arc::new(data);
-        let mut inner = self.inner.lock().unwrap_or_else(|e| e.into_inner());
-        let data_len = data.len();
-
-        // Don't cache if single chunk exceeds budget — still return the data
-        // to the caller, just don't retain it.
-        if data_len > inner.max_bytes {
-            return data;
+        let mut inner = self.lock();
+        let addr = inner.current();
+        inner.put_decompressed((addr, coord), data)
     }
 
-        // Check if already present
-        inner.tick += 1;
-        let tick = inner.tick;
-        if let Some(&idx) = inner.slot_index.get(&coord) {
-            inner.slots[idx].last_access = tick;
-            return Arc::clone(&inner.slots[idx].data); // already cached
+    /// [`Self::prefetch_hint_in`] for the bound dataset.
+    pub fn prefetch_hint(&self, next_coords: &[ChunkCoord]) {
+        let addr = self.lock().current();
+        self.prefetch_hint_in(addr, next_coords);
     }
 
-        // Evict until we have room
-        while inner.slots.len() >= inner.max_slots
-            || (inner.current_bytes + data_len > inner.max_bytes && !inner.slots.is_empty())
-        {
-            // Find LRU slot
-            let lru_idx = inner
-                .slots
-                .iter()
-                .enumerate()
-                .min_by_key(|(_, s)| s.last_access)
-                .map(|(i, _)| i)
-                .unwrap();
-            let removed = inner.slots.swap_remove(lru_idx);
-            inner.slot_index.remove(&removed.coord);
-            // swap_remove moved the former last element into `lru_idx` (unless
-            // it *was* the last element) — fix up that element's index entry.
-            if lru_idx < inner.slots.len() {
-                let moved_coord = inner.slots[lru_idx].coord.clone();
-                inner.slot_index.insert(moved_coord, lru_idx);
-            }
-            inner.current_bytes -= removed.data.len();
-            inner.stats.evictions += 1;
-        }
+    // ----- Whole-cache operations -----
 
-        inner.current_bytes += data_len;
-        let new_idx = inner.slots.len();
-        inner.slot_index.insert(coord.clone(), new_idx);
-        inner.slots.push(CachedChunk {
-            coord,
-            data: Arc::clone(&data),
-            last_access: tick,
-        });
-        data
-    }
-
-    /// Clear the entire cache (index + decompressed data).
+    /// Clear the entire cache (indexes + decompressed data + stats).
     pub fn clear(&self) {
-        let mut inner = self.inner.lock().unwrap_or_else(|e| e.into_inner());
-        inner.index = None;
-        inner.index_addr = None;
+        let mut inner = self.lock();
+        inner.datasets.clear();
+        inner.current = None;
         inner.slots.clear();
         inner.slot_index.clear();
         inner.current_bytes = 0;
         inner.tick = 0;
         inner.last_coord = None;
         inner.stats = AccessStats::default();
-        inner.chunk_index = None;
-        inner.chunk_layout = None;
-    }
-
-    /// Record that the given chunk coordinates are predicted to be accessed
-    /// soon (bookkeeping only).
-    ///
-    /// This does **not** prefetch or pre-decompress anything — it only
-    /// checks whether each coordinate is already in the chunk index and
-    /// updates access-pattern stats accordingly. Real prefetching (e.g.
-    /// background pre-decompression) is not implemented.
-    pub fn prefetch_hint(&self, next_coords: &[ChunkCoord]) {
-        let inner = self.inner.lock().unwrap_or_else(|e| e.into_inner());
-        if inner.index.is_none() {
-            return;
-        }
-        drop(inner);
-        // For each predicted coordinate, verify it exists in the index.
-        // The index is already populated, so this is a no-op for known chunks.
-        // The purpose is to signal intent — callers can pre-decompress if needed.
-        // We touch the stats to record that prefetch hints were issued.
-        let mut inner = self.inner.lock().unwrap_or_else(|e| e.into_inner());
-        for coord in next_coords {
-            let exists = inner
-                .index
-                .as_ref()
-                .map(|idx| idx.contains_key(coord))
-                .unwrap_or(false);
-            if exists {
-                inner.stats.sequential_count += 1;
-            }
-        }
     }
 
     /// Return the current access pattern statistics.
     pub fn access_stats(&self) -> AccessStats {
-        self.inner
-            .lock()
-            .unwrap_or_else(|e| e.into_inner())
-            .stats
-            .clone()
+        self.lock().stats.clone()
     }
 
     /// Update the sweep direction label in the access stats.
     pub fn set_sweep_direction(&self, direction: &'static str) {
-        self.inner
-            .lock()
-            .unwrap_or_else(|e| e.into_inner())
-            .stats
-            .sweep_direction = Some(direction);
+        self.lock().stats.sweep_direction = Some(direction);
     }
 
-    /// Number of decompressed chunks currently cached.
+    /// Number of decompressed chunks currently cached (all datasets).
     pub fn cached_chunk_count(&self) -> usize {
-        self.inner
-            .lock()
-            .unwrap_or_else(|e| e.into_inner())
-            .slots
-            .len()
+        self.lock().slots.len()
     }
 
-    /// Total bytes of decompressed data currently cached.
+    /// Total bytes of decompressed data currently cached (all datasets).
     pub fn cached_bytes(&self) -> usize {
-        self.inner
-            .lock()
-            .unwrap_or_else(|e| e.into_inner())
-            .current_bytes
+        self.lock().current_bytes
+    }
+
+    /// Number of datasets whose chunk index is currently kept.
+    pub fn indexed_dataset_count(&self) -> usize {
+        self.lock().datasets.len()
     }
 }
 
@@ -808,6 +967,92 @@ mod tests {
         assert_eq!(cache.cached_bytes(), 0);
     }
 
+    #[test]
+    fn datasets_sharing_coordinates_stay_separate() {
+        let cache = ChunkCache::new();
+        let a = vec![make_chunk(vec![0, 0], 0x100, 8)];
+        let b = vec![make_chunk(vec![0, 0], 0x900, 8)];
+        let got_a = cache.chunks_for::<()>(1, 1, || Ok(a.clone())).unwrap();
+        let got_b = cache.chunks_for::<()>(2, 1, || Ok(b.clone())).unwrap();
+        assert_eq!(got_a[0].address, 0x100);
+        assert_eq!(got_b[0].address, 0x900);
+        // Built once per dataset: a second lookup doesn't call the builder.
+        let again = cache
+            .chunks_for::<()>(1, 1, || panic!("index rebuilt"))
+            .unwrap();
+        assert_eq!(again[0].address, 0x100);
+
+        cache.put_decompressed_in(1, vec![0], vec![1; 4]);
+        cache.put_decompressed_in(2, vec![0], vec![2; 4]);
+        assert_eq!(
+            cache.get_decompressed_in(1, &[0]).unwrap().as_slice(),
+            &[1; 4]
+        );
+        assert_eq!(
+            cache.get_decompressed_in(2, &[0]).unwrap().as_slice(),
+            &[2; 4]
+        );
+        assert!(cache.get_decompressed_in(3, &[0]).is_none());
+        assert_eq!(cache.cached_chunk_count(), 2);
+
+        // The bound-dataset methods see only the bound dataset.
+        cache.ensure_dataset(2);
+        assert_eq!(cache.lookup_index(&[0]).unwrap().address, 0x900);
+        assert_eq!(cache.get_decompressed(&[0]).unwrap(), vec![2; 4]);
+    }
+
+    #[test]
+    fn dataset_indexes_are_bounded() {
+        let cache = ChunkCache::new();
+        for addr in 0..(MAX_INDEXED_DATASETS as u64 + 10) {
+            cache
+                .chunks_for::<()>(addr, 1, || Ok(vec![make_chunk(vec![0], addr, 8)]))
+                .unwrap();
+        }
+        assert_eq!(cache.indexed_dataset_count(), MAX_INDEXED_DATASETS);
+
+        // One huge index evicts the others but is itself kept.
+        let huge: Vec<ChunkInfo> = (0..MAX_INDEXED_CHUNKS as u64)
+            .map(|i| make_chunk(vec![i], i, 8))
+            .collect();
+        let got = cache.chunks_for::<()>(9999, 1, || Ok(huge)).unwrap();
+        assert_eq!(got.len(), MAX_INDEXED_CHUNKS);
+        assert_eq!(cache.indexed_dataset_count(), 1);
+    }
+
+    #[test]
+    fn concurrent_readers_of_different_datasets_see_their_own_chunks() {
+        let cache = std::sync::Arc::new(ChunkCache::with_capacity(1 << 20, 64));
+        let handles: Vec<_> = (0..8u64)
+            .map(|t| {
+                let cache = std::sync::Arc::clone(&cache);
+                std::thread::spawn(move || {
+                    for round in 0..500u64 {
+                        let addr = (t + round) % 16;
+                        let coord = vec![round % 4];
+                        let chunks = cache
+                            .chunks_for::<()>(addr, 1, || {
+                                Ok((0..4).map(|c| make_chunk(vec![c], addr, 8)).collect())
+                            })
+                            .unwrap();
+                        assert!(chunks.iter().all(|c| c.address == addr));
+                        let want = vec![addr as u8; 8];
+                        let got = match cache.get_decompressed_in(addr, &coord) {
+                            Some(hit) => hit.to_vec(),
+                            None => cache
+                                .put_decompressed_in(addr, coord, want.clone())
+                                .to_vec(),
+                        };
+                        assert_eq!(got, want);
+                    }
+                })
+            })
+            .collect();
+        for h in handles {
+            h.join().unwrap();
+        }
+    }
+
     #[test]
     fn duplicate_insert_is_noop() {
         let cache = ChunkCache::new();

crates/clawhdf5-format/src/chunk_grid.rs

@@ -0,0 +1,200 @@
+//! Chunk-index linearisation shared by the Fixed Array and Extensible Array
+//! chunk indexes (reader and writer).
+//!
+//! Both indexes store one element per chunk at a *linear* index, and the
+//! library derives that index from the chunk's scaled coordinates
+//! (`offset / chunk_dim`) using the dataset's **maximum** dimensions, not its
+//! current ones (`H5D__farray_idx_get_addr` / `H5D__earray_idx_get_addr`,
+//! via `layout->max_down_chunks`). A dataset whose current shape is smaller
+//! than its maxshape therefore has gaps in the index, and laying it out by the
+//! current shape puts every chunk after the first row in the wrong place.
+//!
+//! The Extensible Array adds one more step: its one unlimited dimension has no
+//! finite chunk count, so the library *swizzles* the coordinates to make that
+//! dimension the slowest-varying one (`H5VM_swizzle_coords`, which moves
+//! `coords[unlim_dim]` to the front and shifts the dimensions before it right
+//! by one) before linearising with `swizzled_max_down_chunks`. When the
+//! unlimited dimension is already dimension 0 no swizzle happens.
+
+#[cfg(not(feature = "std"))]
+extern crate alloc;
+
+#[cfg(not(feature = "std"))]
+use alloc::{vec, vec::Vec};
+
+use crate::error::FormatError;
+
+/// How a chunk index maps linear element indexes to chunk coordinates.
+#[derive(Debug, Clone)]
+pub(crate) struct ChunkGrid {
+    /// Spatial chunk dimensions, in dataset order.
+    chunk_dims: Vec<u64>,
+    /// Chunks per dimension covering the *current* extent, in dataset order.
+    cur_chunks: Vec<u64>,
+    /// Dataset dimension stored at each linearisation position (slowest
+    /// first). The identity except for a swizzled Extensible Array.
+    order: Vec<usize>,
+    /// Linear stride of each linearisation position.
+    down: Vec<u64>,
+}
+
+impl ChunkGrid {
+    /// Grid for a Fixed Array index: row-major over the chunk counts of the
+    /// maximum dimensions (`max_dims`, falling back to the current dimensions
+    /// when the dataspace records none).
+    pub(crate) fn fixed_array(
+        cur_dims: &[u64],
+        max_dims: Option<&[u64]>,
+        chunk_dims: &[u64],
+    ) -> Result<Self, FormatError> {
+        Self::build(cur_dims, max_dims, chunk_dims, None)
+    }
+
+    /// Grid for an Extensible Array index: like the Fixed Array, but the
+    /// unlimited dimension (the one whose maximum is `H5S_UNLIMITED`) is moved
+    /// to the slowest-varying position first.
+    pub(crate) fn extensible_array(
+        cur_dims: &[u64],
+        max_dims: Option<&[u64]>,
+        chunk_dims: &[u64],
+    ) -> Result<Self, FormatError> {
+        let unlim = max_dims.and_then(|m| m.iter().position(|&d| d == u64::MAX));
+        Self::build(cur_dims, max_dims, chunk_dims, unlim)
+    }
+
+    fn build(
+        cur_dims: &[u64],
+        max_dims: Option<&[u64]>,
+        chunk_dims: &[u64],
+        unlim: Option<usize>,
+    ) -> Result<Self, FormatError> {
+        let rank = chunk_dims.len();
+        if cur_dims.len() != rank || max_dims.is_some_and(|m| m.len() != rank) {
+            return Err(FormatError::ChunkedReadError(
+                "chunk index rank does not match the dataspace".into(),
+            ));
+        }
+        if chunk_dims.contains(&0) {
+            return Err(FormatError::ChunkedReadError(
+                "chunk dimension is zero".into(),
+            ));
+        }
+        let cur_chunks: Vec<u64> = cur_dims
+            .iter()
+            .zip(chunk_dims)
+            .map(|(&d, &c)| d.div_ceil(c))
+            .collect();
+        // Chunk counts of the maximum extent. An unlimited dimension has no
+        // finite count; it only ever sits in the slowest position, where its
+        // count never enters a stride. A (corrupt) maximum smaller than the
+        // current extent is widened so no allocated chunk becomes unreachable.
+        let max_chunks: Vec<u64> = (0..rank)
+            .map(|d| {
+                let max = max_dims.map_or(cur_dims[d], |m| m[d]);
+                if max == u64::MAX {
+                    u64::MAX
+                } else {
+                    max.div_ceil(chunk_dims[d]).max(cur_chunks[d])
+                }
+            })
+            .collect();
+
+        let mut order: Vec<usize> = (0..rank).collect();
+        if let Some(u) = unlim {
+            order.remove(u);
+            order.insert(0, u);
+        }
+        let mut down = vec![1u64; rank];
+        for p in (0..rank.saturating_sub(1)).rev() {
+            let next = max_chunks[order[p + 1]];
+            if next == u64::MAX {
+                // Only reachable with more than one unlimited dimension, which
+                // neither index type can describe.
+                return Err(FormatError::ChunkedReadError(
+                    "array chunk index with more than one unlimited dimension".into(),
+                ));
+            }
+            down[p] = down[p + 1].checked_mul(next).ok_or_else(|| {
+                FormatError::Overflow("chunk index linear stride overflows u64".into())
+            })?;
+        }
+        Ok(Self {
+            chunk_dims: chunk_dims.to_vec(),
+            cur_chunks,
+            order,
+            down,
+        })
+    }
+
+    /// Dataset-space offsets of the chunk stored at linear `index`, or `None`
+    /// when that chunk lies outside the current extent (the index still has a
+    /// slot for it; the library ignores such chunks on read).
+    pub(crate) fn offsets(&self, index: u64) -> Option<Vec<u64>> {
+        let rank = self.chunk_dims.len();
+        let mut offsets = vec![0u64; rank];
+        let mut rem = index;
+        for p in 0..rank {
+            let d = self.order[p];
+            let scaled = rem / self.down[p];
+            rem %= self.down[p];
+            if scaled >= self.cur_chunks[d] {
+                return None;
+            }
+            offsets[d] = scaled * self.chunk_dims[d];
+        }
+        Some(offsets)
+    }
+
+    /// Linear index of the chunk with scaled coordinates `scaled`
+    /// (`offset / chunk_dim` per dimension, in dataset order).
+    pub(crate) fn linear_index(&self, scaled: &[u64]) -> u64 {
+        self.order
+            .iter()
+            .zip(&self.down)
+            .map(|(&d, &stride)| scaled[d] * stride)
+            .sum()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn fixed_array_uses_max_dims() {
+        // shape (4, 6), chunks (2, 3), maxshape (20, 10): 10 x 4 chunk grid.
+        let g = ChunkGrid::fixed_array(&[4, 6], Some(&[20, 10]), &[2, 3]).unwrap();
+        assert_eq!(g.offsets(0), Some(vec![0, 0]));
+        assert_eq!(g.offsets(1), Some(vec![0, 3]));
+        assert_eq!(g.offsets(2), None); // column chunk 2 is beyond the extent
+        assert_eq!(g.offsets(4), Some(vec![2, 0]));
+        assert_eq!(g.offsets(5), Some(vec![2, 3]));
+        assert_eq!(g.offsets(8), None); // row chunk 2 is beyond the extent
+        assert_eq!(g.linear_index(&[1, 1]), 5);
+    }
+
+    #[test]
+    fn extensible_array_swizzles_unlimited_dim() {
+        // maxshape (10, None): dim 1 is unlimited and becomes slowest.
+        let g = ChunkGrid::extensible_array(&[4, 6], Some(&[10, u64::MAX]), &[2, 3]).unwrap();
+        // max chunks of dim 0 = 5, so index = c1 * 5 + c0.
+        assert_eq!(g.linear_index(&[1, 0]), 1);
+        assert_eq!(g.linear_index(&[0, 1]), 5);
+        assert_eq!(g.offsets(5), Some(vec![0, 3]));
+        assert_eq!(g.offsets(6), Some(vec![2, 3]));
+        assert_eq!(g.offsets(2), None);
+    }
+
+    #[test]
+    fn extensible_array_unlimited_first_is_row_major() {
+        let g = ChunkGrid::extensible_array(&[4, 6], Some(&[u64::MAX, 30]), &[2, 3]).unwrap();
+        // max chunks of dim 1 = 10.
+        assert_eq!(g.linear_index(&[1, 1]), 11);
+        assert_eq!(g.offsets(11), Some(vec![2, 3]));
+    }
+
+    #[test]
+    fn rejects_two_unlimited_dims_after_the_first() {
+        assert!(ChunkGrid::fixed_array(&[4, 6], Some(&[u64::MAX, u64::MAX]), &[2, 3]).is_err());
+    }
+}

crates/clawhdf5-format/src/chunked_read.rs

@@ -15,7 +15,7 @@ use crate::datatype::Datatype;
 use crate::error::FormatError;
 use crate::extensible_array::{ExtensibleArrayHeader, read_extensible_array_chunks};
 use crate::filter_pipeline::FilterPipeline;
-use crate::filters::decompress_chunk;
+use crate::filters::{all_filters_skipped, decompress_chunk_masked};
 use crate::fixed_array::{FixedArrayHeader, read_fixed_array_chunks};
 #[cfg(feature = "std")]
 use std::sync::Arc;
@@ -65,11 +65,13 @@ fn decompress_all_chunks(
         let raw_chunk = &file_data[c_addr..c_addr + size];
 
         let decompressed = if let Some(pl) = pipeline {
-            if chunk_info.filter_mask == 0 {
-                decompress_chunk(raw_chunk, pl, chunk_total_bytes, element_size)?
-            } else {
-                raw_chunk.to_vec()
-            }
+            decompress_chunk_masked(
+                raw_chunk,
+                pl,
+                chunk_total_bytes,
+                element_size,
+                chunk_info.filter_mask,
+            )?
         } else {
             raw_chunk.to_vec()
         };
@@ -223,6 +225,10 @@ pub fn collect_chunk_info(
     collect_chunk_info_inner(file_data, btree_address, ndims, offset_size, length_size, 0)
 }
 
+/// Width of each chunk offset in a v1 chunk B-tree key, independent of the
+/// file's size-of-offsets.
+const CHUNK_KEY_OFFSET_SIZE: u8 = 8;
+
 /// Maximum recursion depth for chunk B-tree traversal (malformed/cyclic data
 /// protection), matching `btree_v1.rs`'s `MAX_BTREE_DEPTH`.
 const MAX_CHUNK_BTREE_DEPTH: usize = 64;
@@ -260,8 +266,14 @@ fn collect_chunk_info_inner(
 
     let mut pos = offset + 8 + os * 2; // skip left/right sibling
 
-    // Key size: chunk_size(4) + filter_mask(4) + ndims * offset_size
-    let key_size = 4 + 4 + ndims * os;
+    // Key: chunk_size(4) + filter_mask(4) + one offset per dimension. The
+    // offsets are always 8 bytes each — they are dataset coordinates, not file
+    // addresses, so they do not follow the superblock's size-of-offsets (only
+    // the sibling and child addresses do).
+    let key_size = ndims
+        .checked_mul(CHUNK_KEY_OFFSET_SIZE as usize)
+        .and_then(|n| n.checked_add(8))
+        .ok_or_else(|| FormatError::ChunkedReadError("chunk key too large".into()))?;
 
     if node_level == 0 {
         // Leaf node: keys and children interleaved
@@ -287,8 +299,8 @@ fn collect_chunk_info_inner(
             let mut offsets = Vec::with_capacity(ndims);
             let mut kp = pos + 8;
             for _ in 0..ndims {
-                offsets.push(read_offset(file_data, kp, offset_size)?);
-                kp += os;
+                offsets.push(read_offset(file_data, kp, CHUNK_KEY_OFFSET_SIZE)?);
+                kp += CHUNK_KEY_OFFSET_SIZE as usize;
             }
             pos += key_size;
 
@@ -507,6 +519,7 @@ pub fn list_chunks(
         addr_opt,
         single_filtered_size,
         single_filter_mask,
+        unfiltered_edges,
     ) = match layout {
         DataLayout::Chunked {
             chunk_dimensions,
@@ -515,6 +528,7 @@ pub fn list_chunks(
             chunk_index_type,
             single_chunk_filtered_size,
             single_chunk_filter_mask,
+            dont_filter_partial_edge_chunks,
         } => (
             chunk_dimensions,
             *version,
@@ -522,6 +536,7 @@ pub fn list_chunks(
             *btree_address,
             *single_chunk_filtered_size,
             *single_chunk_filter_mask,
+            *dont_filter_partial_edge_chunks,
         ),
         _ => {
             return Err(FormatError::ChunkedReadError(
@@ -554,7 +569,7 @@ pub fn list_chunks(
     }
 
     // Collect chunks based on version and index type
-    let chunks = match (version, chunk_index_type) {
+    let mut chunks = match (version, chunk_index_type) {
         (3, _) => {
             let ndims = chunk_dimensions.len(); // rank+1
             collect_chunk_info(file_data, addr, ndims, offset_size, length_size)?
@@ -593,6 +608,7 @@ pub fn list_chunks(
                 file_data,
                 &header,
                 &dataspace.dimensions,
+                dataspace.max_dimensions.as_deref(),
                 spatial_chunk_dims,
                 elem_size as u32,
                 offset_size,
@@ -608,6 +624,7 @@ pub fn list_chunks(
                 file_data,
                 &header,
                 &dataspace.dimensions,
+                dataspace.max_dimensions.as_deref(),
                 spatial_chunk_dims,
                 elem_size as u32,
                 offset_size,
@@ -633,6 +650,23 @@ pub fn list_chunks(
         }
     };
 
+    // With "don't filter partial edge chunks", a chunk that extends past the
+    // dataset's extent is stored raw while its filter mask still reads 0.
+    // Mark every filter skipped so all read paths copy it as-is.
+    if unfiltered_edges {
+        for chunk in &mut chunks {
+            let partial = chunk
+                .offsets
+                .iter()
+                .zip(&chunk_dims)
+                .zip(&ds_dims)
+                .any(|((&off, &cd), &dd)| off.saturating_add(cd as u64) > dd as u64);
+            if partial {
+                chunk.filter_mask = u32::MAX;
+            }
+        }
+    }
+
     Ok((chunks, chunk_dims))
 }
 
@@ -804,24 +838,20 @@ pub fn read_chunked_data_cached(
         )));
     }
 
-    // The per-file cache is shared across datasets; bind it to this one so a
-    // different dataset's chunk index is never reused for this read.
-    cache.ensure_dataset(addr);
-
-    // Populate chunk index on first access
-    if !cache.has_index() {
-        let (chunks, _) = list_chunks(
+    // The per-file cache is shared across datasets (and threads); every
+    // lookup is keyed by this dataset's chunk-index address, so another
+    // dataset's index or chunks are never used for this read.
+    let chunks = cache.chunks_for(addr, rank, || {
+        list_chunks(
             file_data,
             layout,
             dataspace,
             elem_size,
             offset_size,
             length_size,
-        )?;
-        cache.populate_index(&chunks, rank);
-    }
-
-    let chunks = cache.all_indexed_chunks().unwrap_or_default();
+        )
+        .map(|(chunks, _)| chunks)
+    })?;
 
     // Assemble output
     let total_bytes = checked_byte_len(dataspace.checked_num_elements()?, elem_size)?;
@@ -876,10 +906,11 @@ pub fn read_chunked_data_cached(
     };
 
     // Chunks stored as-is (no pipeline, or the filter mask says this chunk
-    // skipped it) are copied straight from the file bytes: they are already in
-    // memory, so routing them through a Vec and then an aligned cache buffer
-    // was two extra copies of the whole dataset for nothing.
-    let stored_raw = |c: &ChunkInfo| pipeline.is_none() || c.filter_mask != 0;
+    // skipped every filter) are copied straight from the file bytes: they are
+    // already in memory, so routing them through a Vec and then an aligned
+    // cache buffer was two extra copies of the whole dataset for nothing.
+    let stored_raw =
+        |c: &ChunkInfo| pipeline.is_none_or(|pl| all_filters_skipped(pl, c.filter_mask));
     let mut misses: Vec<&ChunkInfo> = Vec::new();
     for chunk_info in &chunks {
         if stored_raw(chunk_info) {
@@ -887,7 +918,7 @@ pub fn read_chunked_data_cached(
             continue;
         }
         let coord: Vec<u64> = chunk_info.offsets.iter().take(rank).copied().collect();
-        match cache.get_decompressed_aligned(&coord) {
+        match cache.get_decompressed_in(addr, &coord) {
             Some(cached) => place(&cached, chunk_info),
             None => misses.push(chunk_info),
         }
@@ -901,7 +932,13 @@ pub fn read_chunked_data_cached(
     let cache_them = total_bytes <= cache.max_bytes();
     if let Some(pl) = pipeline {
         let decode = |c: &&ChunkInfo| -> Result<Vec<u8>, FormatError> {
-            decompress_chunk(raw_bytes(c)?, pl, chunk_total_bytes, elem_size as u32)
+            decompress_chunk_masked(
+                raw_bytes(c)?,
+                pl,
+                chunk_total_bytes,
+                elem_size as u32,
+                c.filter_mask,
+            )
         };
         for batch in misses.chunks(DECODE_BATCH) {
             #[cfg(feature = "parallel")]
@@ -918,7 +955,7 @@ pub fn read_chunked_data_cached(
                 let data = data?;
                 if cache_them {
                     let coord: Vec<u64> = chunk_info.offsets.iter().take(rank).copied().collect();
-                    let cached = cache.put_decompressed(coord, data);
+                    let cached = cache.put_decompressed_in(addr, coord, data);
                     place(&cached, chunk_info);
                 } else {
                     place(&data, chunk_info);
@@ -1122,24 +1159,20 @@ pub fn read_chunked_data_sweep(
         )));
     }
 
-    // The per-file cache is shared across datasets; bind it to this one so a
-    // different dataset's chunk index is never reused for this read.
-    cache.ensure_dataset(addr);
-
-    // Populate chunk index on first access
-    if !cache.has_index() {
-        let (chunks, _) = list_chunks(
+    // The per-file cache is shared across datasets (and threads); every
+    // lookup is keyed by this dataset's chunk-index address, so another
+    // dataset's index or chunks are never used for this read.
+    let chunks = cache.chunks_for(addr, rank, || {
+        list_chunks(
             file_data,
             layout,
             dataspace,
             elem_size,
             offset_size,
             length_size,
-        )?;
-        cache.populate_index(&chunks, rank);
-    }
-
-    let chunks = cache.all_indexed_chunks().unwrap_or_default();
+        )
+        .map(|(chunks, _)| chunks)
+    })?;
 
     // Assemble output
     let total_bytes = checked_byte_len(dataspace.checked_num_elements()?, elem_size)?;
@@ -1170,12 +1203,12 @@ pub fn read_chunked_data_sweep(
 
         // Issue prefetch hint for predicted next chunks
         if !sweep.predicted_next.is_empty() {
-            cache.prefetch_hint(&sweep.predicted_next);
+            cache.prefetch_hint_in(addr, &sweep.predicted_next);
             cache.set_sweep_direction(sweep.direction);
         }
 
         // Try decompressed cache first
-        let decompressed = if let Some(cached) = cache.get_decompressed_aligned(&coord) {
+        let decompressed = if let Some(cached) = cache.get_decompressed_in(addr, &coord) {
             cached
         } else {
             // Decompress from file
@@ -1184,15 +1217,17 @@ pub fn read_chunked_data_sweep(
             ensure_len(file_data, c_addr, size)?;
             let raw_chunk = &file_data[c_addr..c_addr + size];
             let dec = if let Some(pl) = pipeline {
-                if chunk_info.filter_mask == 0 {
-                    decompress_chunk(raw_chunk, pl, chunk_total_bytes, elem_size as u32)?
-                } else {
-                    raw_chunk.to_vec()
-                }
+                decompress_chunk_masked(
+                    raw_chunk,
+                    pl,
+                    chunk_total_bytes,
+                    elem_size as u32,
+                    chunk_info.filter_mask,
+                )?
             } else {
                 raw_chunk.to_vec()
             };
-            cache.put_decompressed(coord, dec)
+            cache.put_decompressed_in(addr, coord, dec)
         };
 
         let chunk_offsets: Vec<usize> = chunk_info
@@ -1276,48 +1311,34 @@ pub fn read_chunked_data_indexed(
         )));
     }
 
-    // The per-file cache is shared across datasets; bind it to this one so a
-    // different dataset's chunk index is never reused for this read.
-    cache.ensure_dataset(addr);
-
-    // Build chunk index on first access
-    if !cache.has_chunk_index() {
-        let (chunks, _) = list_chunks(
+    // Chunk index and assembly plan for this dataset, built on first access
+    // and kept per dataset (keyed by chunk-index address) in the shared cache.
+    let plan = cache.chunk_layout_for(
+        addr,
+        rank,
+        || {
+            list_chunks(
                 file_data,
                 layout,
                 dataspace,
                 elem_size,
                 offset_size,
                 length_size,
+            )
+            .map(|(chunks, _)| chunks)
+        },
+        &ds_dims,
+        &chunk_dims,
+        elem_size,
     )?;
-        cache.populate_chunk_index(&chunks, rank);
-        // Also populate the legacy index for compatibility
-        if !cache.has_index() {
-            cache.populate_index(&chunks, rank);
-        }
-    }
-
-    // Build chunk layout on first access
-    if !cache.has_chunk_layout() {
-        cache.populate_chunk_layout(&ds_dims, &chunk_dims, elem_size);
-    }
-
-    // Get the layout info (mappings, output size, chunk total bytes)
-    let (mappings_info, output_bytes, chunk_total_bytes) = cache
-        .with_chunk_layout(|layout| {
-            let info: Vec<_> = layout
-                .mappings
-                .iter()
-                .map(|m| (m.coord.clone(), m.file_offset, m.file_size, m.filter_mask))
-                .collect();
-            (info, layout.output_bytes, layout.chunk_total_bytes)
-        })
-        .ok_or_else(|| FormatError::ChunkedReadError("chunk layout not available".into()))?;
+    let chunk_total_bytes = plan.chunk_total_bytes;
 
     // Decompress chunks (using LRU cache where possible)
-    let mut chunk_buffers: Vec<Arc<CacheAlignedBuffer>> = Vec::with_capacity(mappings_info.len());
-    for (coord, file_offset, file_size, filter_mask) in &mappings_info {
-        if let Some(cached) = cache.get_decompressed_aligned(coord) {
+    let mut chunk_buffers: Vec<Arc<CacheAlignedBuffer>> = Vec::with_capacity(plan.mappings.len());
+    for m in &plan.mappings {
+        let (coord, file_offset, file_size, filter_mask) =
+            (&m.coord, &m.file_offset, &m.file_size, &m.filter_mask);
+        if let Some(cached) = cache.get_decompressed_in(addr, coord) {
             chunk_buffers.push(cached);
         } else {
             let c_addr = *file_offset as usize;
@@ -1325,26 +1346,26 @@ pub fn read_chunked_data_indexed(
             ensure_len(file_data, c_addr, size)?;
             let raw_chunk = &file_data[c_addr..c_addr + size];
             let decompressed = if let Some(pl) = pipeline {
-                if *filter_mask == 0 {
-                    decompress_chunk(raw_chunk, pl, chunk_total_bytes, elem_size as u32)?
-                } else {
-                    raw_chunk.to_vec()
-                }
+                decompress_chunk_masked(
+                    raw_chunk,
+                    pl,
+                    chunk_total_bytes,
+                    elem_size as u32,
+                    *filter_mask,
+                )?
             } else {
                 raw_chunk.to_vec()
             };
             let aligned = CacheAlignedBuffer::from_vec(decompressed);
-            let arc = cache.put_decompressed_aligned(coord.clone(), aligned);
+            let arc = cache.put_decompressed_aligned_in(addr, coord.clone(), aligned);
             chunk_buffers.push(arc);
         }
     }
 
     // Assemble using pre-computed layout
-    let mut output = vec![0u8; output_bytes];
+    let mut output = vec![0u8; plan.output_bytes];
     let data_refs: Vec<&[u8]> = chunk_buffers.iter().map(|b| b.as_slice()).collect();
-    cache.with_chunk_layout(|layout| {
-        layout.assemble(&data_refs, &mut output);
-    });
+    plan.assemble(&data_refs, &mut output);
 
     Ok(output)
 }
@@ -1592,7 +1613,8 @@ mod tests {
                 } else {
                     0
                 };
-                write_offset(&mut buf, off, offset_size);
+                // Key offsets are always 8 bytes (they are coordinates).
+                write_offset(&mut buf, off, 8);
             }
             // Child: address
             write_offset(&mut buf, chunk.address, offset_size);
@@ -1602,7 +1624,7 @@ mod tests {
         buf.extend_from_slice(&0u32.to_le_bytes()); // chunk_size
         buf.extend_from_slice(&0u32.to_le_bytes()); // filter_mask
         for _ in 0..ndims {
-            write_offset(&mut buf, u64::MAX, offset_size);
+            write_offset(&mut buf, u64::MAX, 8);
         }
 
         buf
@@ -1680,6 +1702,37 @@ mod tests {
         assert_eq!(result[2].address, 0x300);
     }
 
+    #[test]
+    fn collect_chunks_with_four_byte_addresses() {
+        // Sibling and child addresses are 4 bytes; the key offsets stay 8.
+        let ndims = 3;
+        let os: u8 = 4;
+        let chunks = vec![
+            ChunkInfo {
+                chunk_size: 80,
+                filter_mask: 2,
+                offsets: vec![0, 5, 0],
+                address: 0x1000,
+            },
+            ChunkInfo {
+                chunk_size: 96,
+                filter_mask: 0,
+                offsets: vec![8, 10, 0],
+                address: 0x2000,
+            },
+        ];
+        let btree = build_chunk_btree_leaf(&chunks, ndims, os);
+        assert_eq!(btree.len(), 8 + 2 * 4 + 2 * (8 + 3 * 8 + 4) + (8 + 3 * 8));
+        let result = collect_chunk_info(&btree, 0, ndims, os, os).unwrap();
+        assert_eq!(result.len(), 2);
+        for (got, want) in result.iter().zip(&chunks) {
+            assert_eq!(got.offsets, want.offsets);
+            assert_eq!(got.address, want.address);
+            assert_eq!(got.chunk_size, want.chunk_size);
+            assert_eq!(got.filter_mask, want.filter_mask);
+        }
+    }
+
     #[test]
     fn collect_empty_btree() {
         let ndims = 2;
@@ -1774,6 +1827,7 @@ mod tests {
             chunk_index_type: None,
             single_chunk_filtered_size: None,
             single_chunk_filter_mask: None,
+            dont_filter_partial_edge_chunks: false,
         };
 
         let dataspace = Dataspace {
@@ -1797,6 +1851,7 @@ mod tests {
             chunk_index_type: None,
             single_chunk_filtered_size: None,
             single_chunk_filter_mask: None,
+            dont_filter_partial_edge_chunks: false,
         };
         let dataspace = Dataspace {
             space_type: DataspaceType::Simple,
@@ -1954,6 +2009,7 @@ mod tests {
             chunk_index_type: None,
             single_chunk_filtered_size: None,
             single_chunk_filter_mask: None,
+            dont_filter_partial_edge_chunks: false,
         };
         let dataspace = Dataspace {
             space_type: DataspaceType::Simple,
@@ -2036,6 +2092,7 @@ mod tests {
             chunk_index_type: None,
             single_chunk_filtered_size: None,
             single_chunk_filter_mask: None,
+            dont_filter_partial_edge_chunks: false,
         };
         let dataspace = Dataspace {
             space_type: DataspaceType::Simple,
@@ -2198,6 +2255,7 @@ mod tests {
             chunk_index_type: Some(1),
             single_chunk_filtered_size: None,
             single_chunk_filter_mask: None,
+            dont_filter_partial_edge_chunks: false,
         };
         let dataspace = Dataspace {
             space_type: DataspaceType::Simple,
@@ -2227,12 +2285,12 @@ mod tests {
         let datatype = make_f64_type();
         let cache = ChunkCache::new();
 
-        assert!(!cache.has_index());
+        assert_eq!(cache.indexed_dataset_count(), 0);
         let raw = read_chunked_data_cached(
             &file_data, &layout, &dataspace, &datatype, None, 8, 8, &cache,
         )
         .unwrap();
-        assert!(cache.has_index());
+        assert_eq!(cache.indexed_dataset_count(), 1);
         assert_eq!(raw.len(), 20 * 8);
         for i in 0..20 {
             let val = f64::from_le_bytes(raw[i * 8..(i + 1) * 8].try_into().unwrap());
@@ -2254,7 +2312,7 @@ mod tests {
             &file_data, &layout, &dataspace, &datatype, None, 8, 8, &cache,
         )
         .unwrap();
-        assert!(cache.has_index());
+        assert_eq!(cache.indexed_dataset_count(), 1);
         assert_eq!(cache.cached_chunk_count(), 0);
 
         // Second read — reuses the cached index
@@ -2263,6 +2321,7 @@ mod tests {
         )
         .unwrap();
         assert_eq!(raw1, raw2);
+        assert_eq!(cache.indexed_dataset_count(), 1);
     }
 
     #[test]

crates/clawhdf5-format/src/chunked_write.rs

@@ -4,15 +4,16 @@
 extern crate alloc;
 
 #[cfg(not(feature = "std"))]
-use alloc::{vec, vec::Vec};
+use alloc::{format, vec, vec::Vec};
 
 use crate::checksum::jenkins_lookup3;
 use crate::chunk_cache::{CACHE_LINE_SIZE, align_to_cache_line};
+use crate::chunk_grid::ChunkGrid;
 use crate::ea_writer;
 use crate::error::FormatError;
 use crate::filter_pipeline::{
-    FILTER_DEFLATE, FILTER_FLETCHER32, FILTER_LZ4, FILTER_PCODEC, FILTER_SHUFFLE, FILTER_ZSTD,
-    FilterDescription, FilterPipeline,
+    FILTER_DEFLATE, FILTER_FLETCHER32, FILTER_LZ4, FILTER_PCODEC, FILTER_PCODEC_NAME,
+    FILTER_SHUFFLE, FILTER_ZSTD, FilterDescription, FilterPipeline,
 };
 use crate::filters::compress_chunk;
 /// Round a file offset up to the next cache-line boundary.
@@ -44,7 +45,8 @@ pub struct ChunkOptions {
     pub lz4: bool,
     /// Zstandard compression level (1-22), None = no zstd. Filter ID 32015.
     pub zstd_level: Option<u32>,
-    /// Pcodec lossless numerical compression. Filter ID 32023.
+    /// Pcodec lossless numerical compression. Private, unregistered filter
+    /// ID [`FILTER_PCODEC`] (480): only clawhdf5 can read it.
     pub pcodec: bool,
 }
 
@@ -115,7 +117,7 @@ impl ChunkOptions {
         if self.pcodec {
             filters.push(FilterDescription {
                 filter_id: FILTER_PCODEC,
-                name: Some("pcodec".into()),
+                name: Some(FILTER_PCODEC_NAME.into()),
                 flags: 0,
                 client_data: vec![element_size],
             });
@@ -443,6 +445,27 @@ fn serialize_v4_fixed_array(
     element_size: u32,
     max_bits: u8,
 ) -> Vec<u8> {
+    let mut buf = layout_v4_chunked_prefix(chunk_dims, element_size);
+
+    // chunk index type = 3 (Fixed Array)
+    buf.push(3);
+
+    // max_dblk_page_nelmts_bits — must match FAHD max_nelmts_bits
+    buf.push(max_bits);
+
+    // Fixed Array header address
+    match offset_size {
+        4 => buf.extend_from_slice(&(fixed_array_address as u32).to_le_bytes()),
+        8 => buf.extend_from_slice(&fixed_array_address.to_le_bytes()),
+        _ => {}
+    }
+
+    buf
+}
+
+/// The part of a v4 chunked layout message before the chunk index type:
+/// version, class, flags and the chunk dimensions (plus the element size).
+fn layout_v4_chunked_prefix(chunk_dims: &[u32], element_size: u32) -> Vec<u8> {
     let mut buf = Vec::new();
     buf.push(4); // version
     buf.push(2); // class = chunked
@@ -482,124 +505,142 @@ fn serialize_v4_fixed_array(
         4 => buf.extend_from_slice(&element_size.to_le_bytes()),
         _ => {}
     }
-
-    // chunk index type = 3 (Fixed Array)
-    buf.push(3);
-
-    // max_dblk_page_nelmts_bits — must match FAHD max_nelmts_bits
-    buf.push(max_bits);
-
-    // Fixed Array header address
-    match offset_size {
-        4 => buf.extend_from_slice(&(fixed_array_address as u32).to_le_bytes()),
-        8 => buf.extend_from_slice(&fixed_array_address.to_le_bytes()),
-        _ => {}
-    }
-
     buf
 }
 
+/// log2 of the elements per Fixed Array data block page (the library's
+/// default, `H5D_FARRAY_MAX_DBLK_PAGE_NELMTS_BITS`).
+const FA_PAGE_BITS: u8 = 10;
+
+pub(crate) fn push_addr(buf: &mut Vec<u8>, addr: u64, offset_size: u8) {
+    match offset_size {
+        4 => buf.extend_from_slice(&(addr as u32).to_le_bytes()),
+        _ => buf.extend_from_slice(&addr.to_le_bytes()),
+    }
+}
+
+/// Width of the chunk-size field of a filtered chunk index element. Must
+/// match the library's `H5D_FARRAY_FILT_COMPUTE_CHUNK_SIZE_LEN` (the EA and
+/// B-tree v2 indexes use the same formula):
+/// `1 + ((log2(unfiltered chunk bytes) + 8) / 8)`, capped at 8.
+pub(crate) fn filtered_chunk_size_len(slots: &[Option<WrittenChunk>]) -> usize {
+    let max_raw = slots
+        .iter()
+        .flatten()
+        .map(|c| c.raw_size)
+        .max()
+        .unwrap_or(1);
+    let log2_val = if max_raw <= 1 {
+        0
+    } else {
+        63 - max_raw.leading_zeros()
+    };
+    (1 + ((log2_val + 8) / 8) as usize).min(8)
+}
+
+/// Append one chunk index element: the chunk's address, plus its stored size
+/// and filter mask when the dataset is filtered. `None` is an unallocated
+/// chunk (undefined address, zero size and mask).
+pub(crate) fn push_index_element(
+    buf: &mut Vec<u8>,
+    slot: Option<&WrittenChunk>,
+    offset_size: u8,
+    chunk_size_bytes: Option<usize>,
+) {
+    match slot {
+        Some(c) => {
+            push_addr(buf, c.address, offset_size);
+            if let Some(n) = chunk_size_bytes {
+                buf.extend_from_slice(&c.compressed_size.to_le_bytes()[..n]);
+                buf.extend_from_slice(&c.filter_mask.to_le_bytes());
+            }
+        }
+        None => {
+            buf.extend(core::iter::repeat_n(0xFF, offset_size as usize));
+            if let Some(n) = chunk_size_bytes {
+                buf.extend(core::iter::repeat_n(0x00, n + 4));
+            }
+        }
+    }
+}
+
 /// Build a complete Fixed Array at a known absolute address.
+///
+/// `slots` holds one entry per element of the array, i.e. per chunk of the
+/// dataset's *maximum* extent in the order [`crate::chunk_grid`] defines;
+/// `None` marks a chunk that is not allocated. An array with more elements
+/// than fit in one page (`2^FA_PAGE_BITS`) gets a paged data block: a
+/// page-init bitmap after the prefix, then one checksummed page per
+/// `2^FA_PAGE_BITS` elements, the last one short (`H5FA__dblock_create`).
 pub fn build_fixed_array_at(
-    chunks: &[WrittenChunk],
+    slots: &[Option<WrittenChunk>],
     offset_size: u8,
     length_size: u8,
     has_filters: bool,
     fa_base_address: u64,
 ) -> Vec<u8> {
     let os = offset_size as usize;
-    let num_elements = chunks.len();
-
-    // For filtered chunks, compute chunk_size encoding width.
-    // Must match the HDF5 C library's H5D_FARRAY_FILT_COMPUTE_CHUNK_SIZE_LEN macro:
-    //   chunk_size_len = 1 + ((H5VM_log2_gen(chunk.size) + 8) / 8)
-    // where chunk.size is the unfiltered chunk size in bytes (product of all chunk dims).
-    let chunk_size_bytes: usize = if has_filters {
-        let max_raw = chunks.iter().map(|c| c.raw_size).max().unwrap_or(1);
-        let log2_val = if max_raw <= 1 {
-            0
-        } else {
-            63 - max_raw.leading_zeros()
-        };
-        let len = 1 + ((log2_val + 8) / 8) as usize;
-        len.min(8)
-    } else {
-        0
-    };
-
-    let elem_size = if has_filters {
-        os + chunk_size_bytes + 4
-    } else {
-        os
-    };
+    let num_elements = slots.len();
 
+    let chunk_size_bytes = has_filters.then(|| filtered_chunk_size_len(slots));
+    let elem_size = os + chunk_size_bytes.map_or(0, |n| n + 4);
     let client_id: u8 = if has_filters { 1 } else { 0 };
 
     // FAHD total size
-    let nelmts_field_size = length_size as usize;
-    let fahd_total_size = 4 + 1 + 1 + 1 + 1 + nelmts_field_size + os + 4;
+    let fahd_total_size = 4 + 1 + 1 + 1 + 1 + length_size as usize + os + 4;
     let fadb_address = fa_base_address + fahd_total_size as u64;
 
-    // Build FAHD
     let mut fahd = Vec::with_capacity(fahd_total_size);
     fahd.extend_from_slice(b"FAHD");
     fahd.push(0); // version
     fahd.push(client_id);
     fahd.push(elem_size as u8);
-
-    // max_nelmts_bits: use 10 as default (page_size = 1024), matching h5py convention
-    let max_bits: u8 = 10;
-    fahd.push(max_bits);
-
+    fahd.push(FA_PAGE_BITS);
     match length_size {
         4 => fahd.extend_from_slice(&(num_elements as u32).to_le_bytes()),
-        8 => fahd.extend_from_slice(&(num_elements as u64).to_le_bytes()),
         _ => fahd.extend_from_slice(&(num_elements as u64).to_le_bytes()),
     }
-
-    match offset_size {
-        4 => fahd.extend_from_slice(&(fadb_address as u32).to_le_bytes()),
-        8 => fahd.extend_from_slice(&fadb_address.to_le_bytes()),
-        _ => fahd.extend_from_slice(&fadb_address.to_le_bytes()),
-    }
-
-    // Checksum
+    push_addr(&mut fahd, fadb_address, offset_size);
     let checksum = jenkins_lookup3(&fahd);
     fahd.extend_from_slice(&checksum.to_le_bytes());
-
     assert_eq!(fahd.len(), fahd_total_size);
 
-    // Build FADB
+    // FADB prefix
     let mut fadb = Vec::new();
     fadb.extend_from_slice(b"FADB");
     fadb.push(0); // version
     fadb.push(client_id);
+    push_addr(&mut fadb, fa_base_address, offset_size);
 
-    // header address
-    match offset_size {
-        4 => fadb.extend_from_slice(&(fa_base_address as u32).to_le_bytes()),
-        8 => fadb.extend_from_slice(&fa_base_address.to_le_bytes()),
-        _ => fadb.extend_from_slice(&fa_base_address.to_le_bytes()),
+    let page_nelmts = 1usize << FA_PAGE_BITS;
+    if num_elements <= page_nelmts {
+        // Unpaged: the elements follow the prefix, one checksum over both.
+        for slot in slots {
+            push_index_element(&mut fadb, slot.as_ref(), offset_size, chunk_size_bytes);
         }
-
-    // Element data
-    for chunk in chunks {
-        match offset_size {
-            4 => fadb.extend_from_slice(&(chunk.address as u32).to_le_bytes()),
-            8 => fadb.extend_from_slice(&chunk.address.to_le_bytes()),
-            _ => fadb.extend_from_slice(&chunk.address.to_le_bytes()),
-        }
-        if has_filters {
-            // Write compressed size using chunk_size_bytes (variable width)
-            let cs_bytes = chunk.compressed_size.to_le_bytes();
-            fadb.extend_from_slice(&cs_bytes[..chunk_size_bytes]);
-            fadb.extend_from_slice(&chunk.filter_mask.to_le_bytes());
-        }
-    }
-
-    // FADB checksum
         let fadb_checksum = jenkins_lookup3(&fadb);
         fadb.extend_from_slice(&fadb_checksum.to_le_bytes());
+    } else {
+        // Paged: every page is written, so every page-init bit is set
+        // (MSB-first, as `H5VM_bit_set` packs them). The prefix and bitmap
+        // share a checksum; each page carries its own.
+        let npages = num_elements.div_ceil(page_nelmts);
+        let mut bitmap = vec![0u8; npages.div_ceil(8)];
+        for p in 0..npages {
+            bitmap[p / 8] |= 0x80 >> (p % 8);
+        }
+        fadb.extend_from_slice(&bitmap);
+        let prefix_checksum = jenkins_lookup3(&fadb);
+        fadb.extend_from_slice(&prefix_checksum.to_le_bytes());
+        for page in slots.chunks(page_nelmts) {
+            let start = fadb.len();
+            for slot in page {
+                push_index_element(&mut fadb, slot.as_ref(), offset_size, chunk_size_bytes);
+            }
+            let page_checksum = jenkins_lookup3(&fadb[start..]);
+            fadb.extend_from_slice(&page_checksum.to_le_bytes());
+        }
+    }
 
     let mut combined = fahd;
     combined.extend_from_slice(&fadb);
@@ -667,7 +708,8 @@ pub fn build_chunked_data_from_precompressed(
     pre: &PrecompressedChunks,
     base_address: u64,
     maxshape: Option<&[u64]>,
-) -> ChunkedDataResult {
+) -> Result<ChunkedDataResult, FormatError> {
+    let index = ChunkIndexPlan::new(&pre.shape, maxshape, &pre.chunk_dims)?;
     let offset_size: u8 = 8;
     let length_size: u8 = 8;
     let num_chunks = pre.chunks.len();
@@ -693,17 +735,18 @@ pub fn build_chunked_data_from_precompressed(
     }
 
     let chunk_dims_u32: Vec<u32> = pre.chunk_dims.iter().map(|&d| d as u32).collect();
-    let use_extensible = maxshape.is_some_and(|ms| ms.contains(&u64::MAX));
 
     let aligned_idx = align_to_cache_line(data_buf.len());
     if aligned_idx > data_buf.len() {
         data_buf.resize(aligned_idx, 0u8);
     }
 
-    let layout_message = if use_extensible {
+    let layout_message = match &index {
+        ChunkIndexPlan::ExtensibleArray(grid) => {
             let ea_address = base_address + data_buf.len() as u64;
+            let slots = index_slots(grid, &pre.shape, &pre.chunk_dims, &written_chunks, None)?;
             let ea_bytes = ea_writer::build_extensible_array_at(
-            &written_chunks,
+                &slots,
                 offset_size,
                 length_size,
                 pre.has_filters,
@@ -716,7 +759,8 @@ pub fn build_chunked_data_from_precompressed(
                 offset_size,
                 element_size as u32,
             )
-    } else if num_chunks == 1 {
+        }
+        ChunkIndexPlan::SingleChunk => {
             let chunk_addr = written_chunks[0].address;
             let filtered_size = if pre.has_filters {
                 Some(written_chunks[0].compressed_size)
@@ -732,10 +776,18 @@ pub fn build_chunked_data_from_precompressed(
                 offset_size,
                 element_size as u32,
             )
-    } else {
+        }
+        ChunkIndexPlan::FixedArray(grid, nslots) => {
             let fa_address = base_address + data_buf.len() as u64;
-        let fa_bytes = build_fixed_array_at(
+            let slots = index_slots(
+                grid,
+                &pre.shape,
+                &pre.chunk_dims,
                 &written_chunks,
+                Some(*nslots),
+            )?;
+            let fa_bytes = build_fixed_array_at(
+                &slots,
                 offset_size,
                 length_size,
                 pre.has_filters,
@@ -747,15 +799,263 @@ pub fn build_chunked_data_from_precompressed(
                 fa_address,
                 offset_size,
                 element_size as u32,
-            10, // max_nelmts_bits — matches h5py convention
+                FA_PAGE_BITS,
             )
+        }
+        ChunkIndexPlan::BTreeV2 => {
+            let bt_address = base_address + data_buf.len() as u64;
+            let records: Vec<(Vec<u64>, &WrittenChunk)> = written_chunks
+                .iter()
+                .enumerate()
+                .map(|(i, c)| (scaled_coords(&pre.shape, &pre.chunk_dims, i), c))
+                .collect();
+            let (bt_bytes, node_size) = build_btree_v2_chunk_index_at(
+                pre.shape.len(),
+                &records,
+                offset_size,
+                length_size,
+                pre.has_filters,
+                bt_address,
+            )?;
+            data_buf.extend_from_slice(&bt_bytes);
+            serialize_v4_btree_v2(
+                &chunk_dims_u32,
+                bt_address,
+                offset_size,
+                element_size as u32,
+                node_size,
+            )
+        }
     };
 
-    ChunkedDataResult {
+    Ok(ChunkedDataResult {
         data_bytes: data_buf,
         layout_message,
         pipeline_message: pre.pipeline_message.clone(),
+    })
+}
+
+/// Most slots a Fixed Array index may have before we refuse to build it: its
+/// data block holds one element per chunk of the *maximum* extent, so a huge
+/// finite maxshape with small chunks would otherwise exhaust memory.
+const MAX_FIXED_ARRAY_SLOTS: u64 = 1 << 26;
+
+/// Which chunk index a dataset gets, following the library's choice in
+/// `H5D__layout_set_latest_indexing`: version-2 B-tree for more than one
+/// unlimited dimension, Extensible Array for exactly one, Fixed Array for a
+/// finite maxshape, Single Chunk when the whole maximum extent is one chunk.
+enum ChunkIndexPlan {
+    SingleChunk,
+    /// The grid and the number of array elements (chunks of the max extent).
+    FixedArray(ChunkGrid, usize),
+    ExtensibleArray(ChunkGrid),
+    BTreeV2,
+}
+
+impl ChunkIndexPlan {
+    fn new(
+        shape: &[u64],
+        maxshape: Option<&[u64]>,
+        chunk_dims: &[u64],
+    ) -> Result<Self, FormatError> {
+        let bad = |what: &str| FormatError::ChunkedReadError(format!("maxshape: {what}"));
+        if let Some(ms) = maxshape {
+            if ms.len() != shape.len() {
+                return Err(bad("rank differs from the shape"));
             }
+            if ms.iter().zip(shape).any(|(&m, &s)| m < s) {
+                return Err(bad("smaller than the shape"));
+            }
+        }
+        let max = maxshape.unwrap_or(shape);
+        let nunlim = max.iter().filter(|&&d| d == u64::MAX).count();
+        match nunlim {
+            0 => {
+                let nslots = max
+                    .iter()
+                    .zip(chunk_dims)
+                    .try_fold(1u64, |acc, (&m, &c)| acc.checked_mul(m.div_ceil(c.max(1))))
+                    .filter(|&n| n <= MAX_FIXED_ARRAY_SLOTS)
+                    .ok_or_else(|| {
+                        bad("too many chunks for a Fixed Array index; \
+                             use larger chunks or an unlimited dimension")
+                    })?;
+                // A Single Chunk index needs that one chunk to exist; an
+                // empty dataset gets an all-unallocated Fixed Array instead.
+                let empty = shape.contains(&0);
+                if nslots == 1 && !empty {
+                    Ok(Self::SingleChunk)
+                } else {
+                    let grid = ChunkGrid::fixed_array(shape, Some(max), chunk_dims)?;
+                    Ok(Self::FixedArray(grid, nslots as usize))
+                }
+            }
+            1 => Ok(Self::ExtensibleArray(ChunkGrid::extensible_array(
+                shape,
+                Some(max),
+                chunk_dims,
+            )?)),
+            _ => Ok(Self::BTreeV2),
+        }
+    }
+}
+
+/// Place each written chunk at its linear index in `grid`. `chunks` are in
+/// row-major order over the chunks of the current extent (`split_into_chunks`).
+/// `len` fixes the slot count (Fixed Array); otherwise it is one past the
+/// highest index used.
+fn index_slots(
+    grid: &ChunkGrid,
+    shape: &[u64],
+    chunk_dims: &[u64],
+    chunks: &[WrittenChunk],
+    len: Option<usize>,
+) -> Result<Vec<Option<WrittenChunk>>, FormatError> {
+    let mut placed: Vec<(usize, &WrittenChunk)> = Vec::with_capacity(chunks.len());
+    for (i, chunk) in chunks.iter().enumerate() {
+        let scaled = scaled_coords(shape, chunk_dims, i);
+        let idx = usize::try_from(grid.linear_index(&scaled))
+            .map_err(|_| FormatError::Overflow("chunk index slot".into()))?;
+        placed.push((idx, chunk));
+    }
+    let n = len.unwrap_or_else(|| placed.iter().map(|&(i, _)| i + 1).max().unwrap_or(0));
+    let mut slots = vec![None; n];
+    for (idx, chunk) in placed {
+        *slots
+            .get_mut(idx)
+            .ok_or_else(|| FormatError::Overflow("chunk index slot".into()))? = Some(chunk.clone());
+    }
+    Ok(slots)
+}
+
+/// Scaled coordinates (`offset / chunk_dim`) of the `i`-th chunk in the
+/// row-major order `split_into_chunks` produces over the current extent.
+fn scaled_coords(shape: &[u64], chunk_dims: &[u64], i: usize) -> Vec<u64> {
+    let rank = shape.len();
+    let mut scaled = vec![0u64; rank];
+    let mut rem = i as u64;
+    for d in (0..rank).rev() {
+        let n = shape[d].div_ceil(chunk_dims[d]);
+        scaled[d] = rem % n;
+        rem /= n;
+    }
+    scaled
+}
+
+/// Node size the library gives a chunk index B-tree (`H5D_BT2_NODE_SIZE`),
+/// with its split and merge percentages.
+const BT2_NODE_SIZE: u32 = 2048;
+const BT2_SPLIT_PERCENT: u8 = 100;
+const BT2_MERGE_PERCENT: u8 = 40;
+/// B-tree v2 record types for chunk indexes (`H5B2_CDSET_ID`,
+/// `H5B2_CDSET_FILT_ID`).
+const BT2_CHUNK_UNFILTERED: u8 = 10;
+const BT2_CHUNK_FILTERED: u8 = 11;
+
+/// Build a version-2 B-tree chunk index (the library's index for datasets
+/// with more than one unlimited dimension) at a known absolute address.
+///
+/// `records` are `(scaled coordinates, chunk)` in lexicographic order of the
+/// coordinates, which is the order the library's comparator
+/// (`H5VM_vector_cmp_u`) keeps them in. The tree is a single leaf: the
+/// library's 2048-byte node when the records fit, otherwise a leaf node
+/// sized to hold them all (the root's record count is 16-bit, so at most
+/// 65535 chunks). Returns the bytes and the node size the layout message
+/// must record.
+fn build_btree_v2_chunk_index_at(
+    rank: usize,
+    records: &[(Vec<u64>, &WrittenChunk)],
+    offset_size: u8,
+    length_size: u8,
+    has_filters: bool,
+    base_address: u64,
+) -> Result<(Vec<u8>, u32), FormatError> {
+    let os = offset_size as usize;
+    let nrec = u16::try_from(records.len()).map_err(|_| {
+        FormatError::ChunkedReadError(
+            "more than 65535 chunks with more than one unlimited dimension: \
+             use larger chunks"
+                .into(),
+        )
+    })?;
+    let chunk_size_bytes = has_filters.then(|| {
+        let slots: Vec<Option<WrittenChunk>> =
+            records.iter().map(|(_, c)| Some((*c).clone())).collect();
+        filtered_chunk_size_len(&slots)
+    });
+    let record_size = os + chunk_size_bytes.map_or(0, |n| n + 4) + 8 * rank;
+    // Leaf: signature, version, type, records, checksum.
+    let leaf_len = 4 + 1 + 1 + records.len() * record_size + 4;
+    let node_size = u32::try_from(leaf_len)
+        .map_err(|_| FormatError::Overflow("B-tree v2 leaf size".into()))?
+        .max(BT2_NODE_SIZE);
+    let tree_type = if has_filters {
+        BT2_CHUNK_FILTERED
+    } else {
+        BT2_CHUNK_UNFILTERED
+    };
+
+    let hdr_len = 4 + 1 + 1 + 4 + 2 + 2 + 1 + 1 + os + 2 + length_size as usize + 4;
+    let leaf_address = base_address + hdr_len as u64;
+
+    let mut out = Vec::with_capacity(hdr_len + node_size as usize);
+    out.extend_from_slice(b"BTHD");
+    out.push(0); // version
+    out.push(tree_type);
+    out.extend_from_slice(&node_size.to_le_bytes());
+    out.extend_from_slice(&(record_size as u16).to_le_bytes());
+    out.extend_from_slice(&0u16.to_le_bytes()); // depth
+    out.push(BT2_SPLIT_PERCENT);
+    out.push(BT2_MERGE_PERCENT);
+    if records.is_empty() {
+        out.extend(core::iter::repeat_n(0xFF, os));
+    } else {
+        push_addr(&mut out, leaf_address, offset_size);
+    }
+    out.extend_from_slice(&nrec.to_le_bytes());
+    match length_size {
+        4 => out.extend_from_slice(&(records.len() as u32).to_le_bytes()),
+        _ => out.extend_from_slice(&(records.len() as u64).to_le_bytes()),
+    }
+    let sum = jenkins_lookup3(&out);
+    out.extend_from_slice(&sum.to_le_bytes());
+    debug_assert_eq!(out.len(), hdr_len);
+    if records.is_empty() {
+        return Ok((out, node_size));
+    }
+
+    let leaf_start = out.len();
+    out.extend_from_slice(b"BTLF");
+    out.push(0); // version
+    out.push(tree_type);
+    for (scaled, chunk) in records {
+        push_index_element(&mut out, Some(chunk), offset_size, chunk_size_bytes);
+        for &c in scaled {
+            out.extend_from_slice(&c.to_le_bytes());
+        }
+    }
+    let sum = jenkins_lookup3(&out[leaf_start..]);
+    out.extend_from_slice(&sum.to_le_bytes());
+    // The library reads whole nodes; pad the leaf out to the node size.
+    out.resize(leaf_start + node_size as usize, 0);
+    Ok((out, node_size))
+}
+
+/// Serialize a v4 layout message for a version-2 B-tree chunk index.
+fn serialize_v4_btree_v2(
+    chunk_dims: &[u32],
+    btree_address: u64,
+    offset_size: u8,
+    element_size: u32,
+    node_size: u32,
+) -> Vec<u8> {
+    let mut buf = layout_v4_chunked_prefix(chunk_dims, element_size);
+    buf.push(5); // chunk index type = 5 (version-2 B-tree)
+    buf.extend_from_slice(&node_size.to_le_bytes());
+    buf.push(BT2_SPLIT_PERCENT);
+    buf.push(BT2_MERGE_PERCENT);
+    push_addr(&mut buf, btree_address, offset_size);
+    buf
 }
 
 /// Build chunked data with absolute addresses.
@@ -790,11 +1090,7 @@ pub fn build_chunked_data_at_ext(
     maxshape: Option<&[u64]>,
 ) -> Result<ChunkedDataResult, FormatError> {
     let pre = precompress_chunks(raw_data, shape, chunk_dims, element_size, options)?;
-    Ok(build_chunked_data_from_precompressed(
-        &pre,
-        base_address,
-        maxshape,
-    ))
+    build_chunked_data_from_precompressed(&pre, base_address, maxshape)
 }
 
 /// Write selected elements into an existing in-memory dataset buffer.
@@ -1314,6 +1610,7 @@ mod tests {
                 chunk_index_type,
                 single_chunk_filtered_size,
                 single_chunk_filter_mask,
+                ..
             } => {
                 assert_eq!(version, 4);
                 assert_eq!(chunk_index_type, Some(1));
@@ -1382,7 +1679,8 @@ mod tests {
                 filter_mask: 0,
             },
         ];
-        let fa = build_fixed_array_at(&chunks, 8, 8, false, 0x2000);
+        let slots: Vec<_> = chunks.into_iter().map(Some).collect();
+        let fa = build_fixed_array_at(&slots, 8, 8, false, 0x2000);
         // Should start with FAHD
         assert_eq!(&fa[0..4], b"FAHD");
         // FAHD size = 4+1+1+1+1+8+8+4 = 28
@@ -1429,7 +1727,8 @@ mod tests {
                 filter_mask: 0,
             },
         ];
-        let ea = ea_writer::build_extensible_array_at(&chunks, 8, 8, false, 0x2000);
+        let slots: Vec<_> = chunks.into_iter().map(Some).collect();
+        let ea = ea_writer::build_extensible_array_at(&slots, 8, 8, false, 0x2000);
         assert_eq!(&ea[0..4], b"EAHD");
         // Find EAIB after EAHD: 12 fixed + 6*8 stats + 8 addr + 4 checksum = 72
         let aehd_size = 4 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 6 * 8 + 8 + 4;

crates/clawhdf5-format/src/data_layout.rs

@@ -53,6 +53,11 @@ pub enum DataLayout {
         single_chunk_filtered_size: Option<u64>,
         /// Filter mask for v4 single chunk with filters.
         single_chunk_filter_mask: Option<u32>,
+        /// Layout v4 flag bit 0 (`H5D_CHUNK_DONT_FILTER_PARTIAL_CHUNKS`):
+        /// partial edge chunks — those extending past the dataset's current
+        /// extent in some dimension — are stored without the filter pipeline,
+        /// even though their filter mask is 0. Always `false` for v3.
+        dont_filter_partial_edge_chunks: bool,
     },
     /// Virtual dataset layout (v4 only).
     Virtual {
@@ -322,6 +327,7 @@ impl DataLayout {
                     chunk_index_type: None,
                     single_chunk_filtered_size: None,
                     single_chunk_filter_mask: None,
+                    dont_filter_partial_edge_chunks: false,
                 })
             }
             _ => Err(FormatError::InvalidLayoutClass(layout_class)),
@@ -505,6 +511,7 @@ impl DataLayout {
                     chunk_index_type: Some(chunk_index_type),
                     single_chunk_filtered_size,
                     single_chunk_filter_mask,
+                    dont_filter_partial_edge_chunks: flags & 0x01 != 0,
                 })
             }
             3 => {
@@ -602,6 +609,7 @@ mod tests {
                 chunk_index_type: None,
                 single_chunk_filtered_size: None,
                 single_chunk_filter_mask: None,
+                dont_filter_partial_edge_chunks: false,
             }
         );
     }
@@ -679,10 +687,35 @@ mod tests {
                 chunk_index_type: Some(1),
                 single_chunk_filtered_size: None,
                 single_chunk_filter_mask: None,
+                dont_filter_partial_edge_chunks: false,
             }
         );
     }
 
+    #[test]
+    fn v4_chunked_dont_filter_partial_edge_chunks_flag() {
+        let mut buf = vec![4u8, 2]; // version=4, class=2
+        buf.push(0x01); // flags bit 0 = don't filter partial edge chunks
+        buf.push(2); // dimensionality=2
+        buf.push(4); // dim_size_encoded_length=4
+        buf.extend_from_slice(&5u32.to_le_bytes());
+        buf.extend_from_slice(&4u32.to_le_bytes());
+        buf.push(3); // Fixed Array
+        buf.push(10); // max_dblk_page_nelmts_bits
+        buf.extend_from_slice(&0x3000u64.to_le_bytes());
+        match DataLayout::parse(&buf, 8, 8).unwrap() {
+            DataLayout::Chunked {
+                dont_filter_partial_edge_chunks,
+                btree_address,
+                ..
+            } => {
+                assert!(dont_filter_partial_edge_chunks);
+                assert_eq!(btree_address, Some(0x3000));
+            }
+            other => panic!("expected Chunked, got {other:?}"),
+        }
+    }
+
     #[test]
     fn v4_chunked_single_chunk_with_filters() {
         let mut buf = vec![4u8, 2]; // version=4, class=2
@@ -705,6 +738,7 @@ mod tests {
                 chunk_index_type: Some(1),
                 single_chunk_filtered_size: Some(1024),
                 single_chunk_filter_mask: Some(0),
+                dont_filter_partial_edge_chunks: false,
             }
         );
     }

crates/clawhdf5-format/src/data_read.rs

@@ -773,14 +773,7 @@ pub fn read_as_f64_zerocopy<'a>(raw: &'a [u8], datatype: &Datatype) -> Option<&'
     // Only native LE f64 is eligible
     #[cfg(target_endian = "little")]
     {
-        if !matches!(
-            datatype,
-            Datatype::FloatingPoint {
-                size: 8,
-                byte_order: DatatypeByteOrder::LittleEndian,
-                ..
-            }
-        ) {
+        if !is_native_le_float(datatype, FloatFormat::Double) {
             return None;
         }
         if !raw.len().is_multiple_of(8) {
@@ -809,14 +802,7 @@ pub fn read_as_f64_zerocopy<'a>(raw: &'a [u8], datatype: &Datatype) -> Option<&'
 pub fn read_as_f32_zerocopy<'a>(raw: &'a [u8], datatype: &Datatype) -> Option<&'a [f32]> {
     #[cfg(target_endian = "little")]
     {
-        if !matches!(
-            datatype,
-            Datatype::FloatingPoint {
-                size: 4,
-                byte_order: DatatypeByteOrder::LittleEndian,
-                ..
-            }
-        ) {
+        if !is_native_le_float(datatype, FloatFormat::Single) {
             return None;
         }
         if !raw.len().is_multiple_of(4) {
@@ -902,9 +888,9 @@ fn native_le_to_vec<T: Copy>(raw: &[u8], count: usize) -> Vec<T> {
 
 /// Convert raw bytes to `f64` values.
 pub fn read_as_f64(raw: &[u8], datatype: &Datatype) -> Result<Vec<f64>, FormatError> {
-    // Array datatypes (e.g. an array-typed compound member) are read as a flat
-    // sequence of their base elements.
-    if let Datatype::Array { base_type, .. } = datatype {
+    // Array datatypes read as a flat sequence of their base elements, and
+    // enumerations (h5py's bool among them) as their integer values.
+    if let Datatype::Array { base_type, .. } | Datatype::Enumeration { base_type, .. } = datatype {
         return read_as_f64(raw, base_type);
     }
     ensure_numeric(datatype, "FloatingPoint or FixedPoint")?;
@@ -919,20 +905,19 @@ pub fn read_as_f64(raw: &[u8], datatype: &Datatype) -> Result<Vec<f64>, FormatEr
 
     // Fast path: native-endian f64 — single bulk memcpy
     #[cfg(target_endian = "little")]
-    if matches!(
-        datatype,
-        Datatype::FloatingPoint {
-            size: 8,
-            byte_order: DatatypeByteOrder::LittleEndian,
-            ..
-        }
-    ) {
+    if is_native_le_float(datatype, FloatFormat::Double) {
         return Ok(native_le_to_vec::<f64>(raw, count));
     }
 
     let order = get_byte_order(datatype);
     let mut result = Vec::with_capacity(count);
-
+    if let Datatype::FloatingPoint { .. } = datatype {
+        let format = FloatFormat::of(datatype)?;
+        for chunk in raw.chunks_exact(elem_size) {
+            result.push(format.decode(chunk, &order));
+        }
+        return Ok(result);
+    }
     for i in 0..count {
         let chunk = &raw[i * elem_size..(i + 1) * elem_size];
         let val = convert_to_f64(chunk, datatype, &order)?;
@@ -947,18 +932,7 @@ fn convert_to_f64(
     order: &DatatypeByteOrder,
 ) -> Result<f64, FormatError> {
     match dt {
-        Datatype::FloatingPoint { size, .. } => match size {
-            4 => {
-                let v = read_f32_bytes(bytes, order);
-                Ok(v as f64)
-            }
-            8 => Ok(read_f64_bytes(bytes, order)),
-            2 => Ok(read_f16_bytes(bytes, order) as f64),
-            _ => Err(FormatError::DataSizeMismatch {
-                expected: 8,
-                actual: *size as usize,
-            }),
-        },
+        Datatype::FloatingPoint { .. } => Ok(FloatFormat::of(dt)?.decode(bytes, order)),
         Datatype::FixedPoint {
             size,
             signed,
@@ -982,9 +956,83 @@ fn convert_to_f64(
     }
 }
 
+/// One numeric element as stored, before conversion to the caller's type.
+#[derive(Debug, Clone, Copy, PartialEq)]
+enum Scalar {
+    Signed(i64),
+    Unsigned(u64),
+    Float(f64),
+}
+
+impl Scalar {
+    // Every conversion follows libhdf5's default (hard) conversions: a value
+    // outside the target type's range saturates to its minimum or maximum —
+    // including a negative value read as unsigned, which reads as 0 — rather
+    // than being truncated to its low bits. Floats truncate toward zero; NaN
+    // converts to 0 (libhdf5 leaves that case to the C cast, whose result is
+    // platform-dependent).
+
+    fn to_i64(self) -> i64 {
+        match self {
+            Scalar::Signed(v) => v,
+            Scalar::Unsigned(v) => i64::try_from(v).unwrap_or(i64::MAX),
+            Scalar::Float(v) => v as i64,
+        }
+    }
+
+    fn to_u64(self) -> u64 {
+        match self {
+            Scalar::Signed(v) => u64::try_from(v).unwrap_or(0),
+            Scalar::Unsigned(v) => v,
+            Scalar::Float(v) => v as u64,
+        }
+    }
+
+    fn to_i32(self) -> i32 {
+        match self {
+            Scalar::Signed(v) => v.clamp(i32::MIN.into(), i32::MAX.into()) as i32,
+            Scalar::Unsigned(v) => i32::try_from(v).unwrap_or(i32::MAX),
+            Scalar::Float(v) => v as i32,
+        }
+    }
+}
+
+/// Decode one element of a numeric datatype.
+fn decode_scalar(
+    bytes: &[u8],
+    dt: &Datatype,
+    order: &DatatypeByteOrder,
+) -> Result<Scalar, FormatError> {
+    match dt {
+        Datatype::FixedPoint {
+            size,
+            signed,
+            bit_offset,
+            bit_precision,
+            ..
+        } => {
+            let full = read_unsigned_int(bytes, *size as usize, order);
+            let (off, prec) = effective_bits(*size as usize, *bit_offset, *bit_precision);
+            Ok(if *signed {
+                Scalar::Signed(extract_signed(full, off, prec))
+            } else {
+                Scalar::Unsigned(extract_unsigned(full, off, prec))
+            })
+        }
+        _ => convert_to_f64(bytes, dt, order).map(Scalar::Float),
+    }
+}
+
 /// Convert raw bytes to `i64` values.
+///
+/// Values are converted the way libhdf5 converts them: integers outside the
+/// target range saturate at its minimum or maximum (a negative value read as
+/// unsigned is 0), and floating-point data is truncated toward zero and
+/// saturated, with NaN read as 0.
 pub fn read_as_i64(raw: &[u8], datatype: &Datatype) -> Result<Vec<i64>, FormatError> {
-    if let Datatype::Array { base_type, .. } = datatype {
+    // Array datatypes read as a flat sequence of their base elements, and
+    // enumerations (h5py's bool among them) as their integer values.
+    if let Datatype::Array { base_type, .. } | Datatype::Enumeration { base_type, .. } = datatype {
         return read_as_i64(raw, base_type);
     }
     ensure_numeric(datatype, "FixedPoint (signed)")?;
@@ -1014,19 +1062,24 @@ pub fn read_as_i64(raw: &[u8], datatype: &Datatype) -> Result<Vec<i64>, FormatEr
     }
 
     let order = get_byte_order(datatype);
-    let (off, prec) = fixed_bits(datatype);
     let mut result = Vec::with_capacity(count);
     for i in 0..count {
         let chunk = &raw[i * elem_size..(i + 1) * elem_size];
-        let full = read_unsigned_int(chunk, elem_size, &order);
-        result.push(extract_signed(full, off, prec));
+        result.push(decode_scalar(chunk, datatype, &order)?.to_i64());
     }
     Ok(result)
 }
 
 /// Convert raw bytes to `u64` values.
+///
+/// Values are converted the way libhdf5 converts them: integers outside the
+/// target range saturate at its minimum or maximum (a negative value read as
+/// unsigned is 0), and floating-point data is truncated toward zero and
+/// saturated, with NaN read as 0.
 pub fn read_as_u64(raw: &[u8], datatype: &Datatype) -> Result<Vec<u64>, FormatError> {
-    if let Datatype::Array { base_type, .. } = datatype {
+    // Array datatypes read as a flat sequence of their base elements, and
+    // enumerations (h5py's bool among them) as their integer values.
+    if let Datatype::Array { base_type, .. } | Datatype::Enumeration { base_type, .. } = datatype {
         return read_as_u64(raw, base_type);
     }
     ensure_numeric(datatype, "FixedPoint (unsigned)")?;
@@ -1039,19 +1092,19 @@ pub fn read_as_u64(raw: &[u8], datatype: &Datatype) -> Result<Vec<u64>, FormatEr
     }
     let count = raw.len() / elem_size;
     let order = get_byte_order(datatype);
-    let (off, prec) = fixed_bits(datatype);
     let mut result = Vec::with_capacity(count);
     for i in 0..count {
         let chunk = &raw[i * elem_size..(i + 1) * elem_size];
-        let full = read_unsigned_int(chunk, elem_size, &order);
-        result.push(extract_unsigned(full, off, prec));
+        result.push(decode_scalar(chunk, datatype, &order)?.to_u64());
     }
     Ok(result)
 }
 
 /// Convert raw bytes to `f32` values.
 pub fn read_as_f32(raw: &[u8], datatype: &Datatype) -> Result<Vec<f32>, FormatError> {
-    if let Datatype::Array { base_type, .. } = datatype {
+    // Array datatypes read as a flat sequence of their base elements, and
+    // enumerations (h5py's bool among them) as their integer values.
+    if let Datatype::Array { base_type, .. } | Datatype::Enumeration { base_type, .. } = datatype {
         return read_as_f32(raw, base_type);
     }
     ensure_numeric(datatype, "FloatingPoint")?;
@@ -1066,31 +1119,36 @@ pub fn read_as_f32(raw: &[u8], datatype: &Datatype) -> Result<Vec<f32>, FormatEr
 
     // Fast path: native-endian f32 — single bulk memcpy
     #[cfg(target_endian = "little")]
-    if matches!(
-        datatype,
-        Datatype::FloatingPoint {
-            size: 4,
-            byte_order: DatatypeByteOrder::LittleEndian,
-            ..
-        }
-    ) {
+    if is_native_le_float(datatype, FloatFormat::Single) {
         return Ok(native_le_to_vec::<f32>(raw, count));
     }
+    // Little-endian IEEE half precision (numpy float16): widen directly.
+    if is_native_le_float(datatype, FloatFormat::Half) {
+        let (halves, _) = raw[..count * 2].as_chunks::<2>();
+        return Ok(halves
+            .iter()
+            .map(|&b| f16_bits_to_f32(u16::from_le_bytes(b)))
+            .collect());
+    }
 
     let order = get_byte_order(datatype);
     let mut result = Vec::with_capacity(count);
+    if let Datatype::FloatingPoint { .. } = datatype {
+        let format = FloatFormat::of(datatype)?;
+        for chunk in raw.chunks_exact(elem_size) {
+            result.push(match format {
+                FloatFormat::Single => read_f32_bytes(chunk, &order),
+                FloatFormat::Half => read_f16_bytes(chunk, &order),
+                // Double rounds; every other supported layout (bfloat16, FP8)
+                // is exact in f32.
+                _ => format.decode(chunk, &order) as f32,
+            });
+        }
+        return Ok(result);
+    }
     for i in 0..count {
         let chunk = &raw[i * elem_size..(i + 1) * elem_size];
         match datatype {
-            Datatype::FloatingPoint { size: 4, .. } => {
-                result.push(read_f32_bytes(chunk, &order));
-            }
-            Datatype::FloatingPoint { size: 8, .. } => {
-                result.push(read_f64_bytes(chunk, &order) as f32);
-            }
-            Datatype::FloatingPoint { size: 2, .. } => {
-                result.push(read_f16_bytes(chunk, &order));
-            }
             Datatype::FixedPoint {
                 signed: true,
                 size,
@@ -1125,8 +1183,15 @@ pub fn read_as_f32(raw: &[u8], datatype: &Datatype) -> Result<Vec<f32>, FormatEr
 }
 
 /// Convert raw bytes to `i32` values.
+///
+/// Values are converted the way libhdf5 converts them: integers outside the
+/// target range saturate at its minimum or maximum (a negative value read as
+/// unsigned is 0), and floating-point data is truncated toward zero and
+/// saturated, with NaN read as 0.
 pub fn read_as_i32(raw: &[u8], datatype: &Datatype) -> Result<Vec<i32>, FormatError> {
-    if let Datatype::Array { base_type, .. } = datatype {
+    // Array datatypes read as a flat sequence of their base elements, and
+    // enumerations (h5py's bool among them) as their integer values.
+    if let Datatype::Array { base_type, .. } | Datatype::Enumeration { base_type, .. } = datatype {
         return read_as_i32(raw, base_type);
     }
     ensure_numeric(datatype, "FixedPoint")?;
@@ -1147,6 +1212,7 @@ pub fn read_as_i32(raw: &[u8], datatype: &Datatype) -> Result<Vec<i32>, FormatEr
             datatype,
             Datatype::FixedPoint {
                 byte_order: DatatypeByteOrder::LittleEndian,
+                signed: true,
                 ..
             }
         )
@@ -1155,12 +1221,10 @@ pub fn read_as_i32(raw: &[u8], datatype: &Datatype) -> Result<Vec<i32>, FormatEr
     }
 
     let order = get_byte_order(datatype);
-    let (off, prec) = fixed_bits(datatype);
     let mut result = Vec::with_capacity(count);
     for i in 0..count {
         let chunk = &raw[i * elem_size..(i + 1) * elem_size];
-        let full = read_unsigned_int(chunk, elem_size, &order);
-        result.push(extract_signed(full, off, prec) as i32);
+        result.push(decode_scalar(chunk, datatype, &order)?.to_i32());
     }
     Ok(result)
 }
@@ -1601,6 +1665,174 @@ fn reorder_bytes(bytes: &[u8], order: &DatatypeByteOrder) -> [u8; 8] {
     buf
 }
 
+/// How the bits of a floating-point datatype are laid out, read from the
+/// datatype message's fields rather than assumed from its size (a 2-byte
+/// float may be IEEE half or bfloat16).
+#[derive(Debug, Clone, Copy, PartialEq)]
+enum FloatFormat {
+    /// IEEE-754 binary16.
+    Half,
+    /// IEEE-754 binary32.
+    Single,
+    /// IEEE-754 binary64.
+    Double,
+    /// Any other IEEE-style layout (implied leading mantissa bit, all-ones
+    /// exponent for infinity/NaN) whose values are all exact in `f64`:
+    /// bfloat16, the FP8 formats, and similar.
+    Other(FloatLayout),
+}
+
+#[derive(Debug, Clone, Copy, PartialEq)]
+struct FloatLayout {
+    exponent_location: u32,
+    exponent_size: u32,
+    mantissa_location: u32,
+    mantissa_size: u32,
+    exponent_bias: u32,
+}
+
+impl FloatFormat {
+    fn of(dt: &Datatype) -> Result<FloatFormat, FormatError> {
+        let Datatype::FloatingPoint {
+            size,
+            exponent_location,
+            exponent_size,
+            mantissa_location,
+            mantissa_size,
+            exponent_bias,
+            ..
+        } = dt
+        else {
+            return Err(FormatError::TypeMismatch {
+                expected: "FloatingPoint",
+                actual: datatype_name(dt),
+            });
+        };
+        let layout = FloatLayout {
+            exponent_location: u32::from(*exponent_location),
+            exponent_size: u32::from(*exponent_size),
+            mantissa_location: u32::from(*mantissa_location),
+            mantissa_size: u32::from(*mantissa_size),
+            exponent_bias: *exponent_bias,
+        };
+        let fields = (
+            layout.exponent_location,
+            layout.exponent_size,
+            layout.mantissa_location,
+            layout.mantissa_size,
+            layout.exponent_bias,
+        );
+        let bits = size.saturating_mul(8);
+        // The sign bit is not kept in `Datatype`; every standard layout has it
+        // directly above the exponent, with the mantissa below.
+        let well_formed = layout.exponent_size > 0
+            && layout.mantissa_size > 0
+            && layout.mantissa_location + layout.mantissa_size <= layout.exponent_location
+            && layout.exponent_location + layout.exponent_size < bits;
+        match (size, fields) {
+            (2, (10, 5, 0, 10, 15)) => Ok(FloatFormat::Half),
+            (4, (23, 8, 0, 23, 127)) => Ok(FloatFormat::Single),
+            (8, (52, 11, 0, 52, 1023)) => Ok(FloatFormat::Double),
+            _ if well_formed
+                && *size <= 8
+                && layout.exponent_size <= 11
+                && layout.mantissa_size <= 52 =>
+            {
+                Ok(FloatFormat::Other(layout))
+            }
+            // Fields that cannot describe any float (e.g. left zeroed by a
+            // hand-built datatype): fall back to the IEEE type of that size.
+            (2, _) if !well_formed => Ok(FloatFormat::Half),
+            (4, _) if !well_formed => Ok(FloatFormat::Single),
+            (8, _) if !well_formed => Ok(FloatFormat::Double),
+            // x87 80-bit extended, binary128, ...: not representable in f64.
+            _ => Err(FormatError::TypeMismatch {
+                expected: "floating point of at most 64 bits (IEEE-style layout)",
+                actual: "FloatingPoint",
+            }),
+        }
+    }
+
+    fn decode(self, bytes: &[u8], order: &DatatypeByteOrder) -> f64 {
+        match self {
+            FloatFormat::Half => f64::from(read_f16_bytes(bytes, order)),
+            FloatFormat::Single => f64::from(read_f32_bytes(bytes, order)),
+            FloatFormat::Double => read_f64_bytes(bytes, order),
+            FloatFormat::Other(layout) => {
+                layout.decode(read_unsigned_int(bytes, bytes.len(), order))
+            }
+        }
+    }
+}
+
+impl FloatLayout {
+    /// Decode the value held in the low `size * 8` bits of `bits`.
+    fn decode(self, bits: u64) -> f64 {
+        let field = |location: u32, size: u32| (bits >> location) & ((1u64 << size) - 1);
+        let exponent = field(self.exponent_location, self.exponent_size);
+        let mantissa = field(self.mantissa_location, self.mantissa_size);
+        let negative = field(self.exponent_location + self.exponent_size, 1) == 1;
+        let max_exponent = (1u64 << self.exponent_size) - 1;
+        let magnitude = if exponent == max_exponent {
+            if mantissa == 0 {
+                f64::INFINITY
+            } else {
+                f64::NAN
+            }
+        } else {
+            let bias = i64::from(self.exponent_bias);
+            let msize = i64::from(self.mantissa_size);
+            // value = significand * 2^power, with an implied leading 1 unless
+            // the number is subnormal (exponent field 0).
+            let (significand, power) = if exponent == 0 {
+                (mantissa, 1 - bias - msize)
+            } else {
+                (
+                    mantissa | (1u64 << self.mantissa_size),
+                    exponent as i64 - bias - msize,
+                )
+            };
+            scale_by_pow2(significand as f64, power)
+        };
+        if negative { -magnitude } else { magnitude }
+    }
+}
+
+/// `x * 2^power` without `std` (no `powi`/`libm`). `x` is a non-negative
+/// integer below 2^53, so it is exact.
+fn scale_by_pow2(x: f64, power: i64) -> f64 {
+    if x == 0.0 || power < -1200 {
+        return 0.0;
+    }
+    if power > 1100 {
+        return f64::INFINITY;
+    }
+    let pow2 = |p: i64| f64::from_bits(((p + 1023) as u64) << 52);
+    let mut x = x;
+    let mut power = power;
+    while power > 1023 {
+        x *= pow2(1023);
+        power -= 1023;
+    }
+    while power < -1022 {
+        x *= pow2(-1022);
+        power += 1022;
+    }
+    x * pow2(power)
+}
+
+/// Whether `datatype` is the little-endian IEEE float `format`, whose bytes
+/// can be copied straight into native values on a little-endian target.
+fn is_native_le_float(datatype: &Datatype, format: FloatFormat) -> bool {
+    matches!(
+        datatype,
+        Datatype::FloatingPoint {
+            byte_order: DatatypeByteOrder::LittleEndian,
+            ..
+        }
+    ) && FloatFormat::of(datatype).is_ok_and(|f| f == format)
+}
+
 fn read_f64_bytes(bytes: &[u8], order: &DatatypeByteOrder) -> f64 {
     let buf = reorder_bytes(bytes, order);
     f64::from_le_bytes(buf)
@@ -1622,36 +1854,7 @@ fn read_f16_bytes(bytes: &[u8], order: &DatatypeByteOrder) -> f32 {
     f16_bits_to_f32(u16::from_le_bytes(buf))
 }
 
-/// Convert the bit pattern of an IEEE-754 half (binary16) to an `f32`.
-fn f16_bits_to_f32(h: u16) -> f32 {
-    let h = h as u32;
-    let sign = (h & 0x8000) << 16;
-    let exp = (h >> 10) & 0x1f;
-    let mant = h & 0x3ff;
-    let bits = if exp == 0 {
-        if mant == 0 {
-            sign // signed zero
-        } else {
-            // Subnormal: normalize into an f32 normal.
-            let mut e: i32 = -1;
-            let mut m = mant;
-            loop {
-                e += 1;
-                m <<= 1;
-                if m & 0x400 != 0 {
-                    break;
-                }
-            }
-            let m = m & 0x3ff;
-            sign | (((127 - 15 - e) as u32) << 23) | (m << 13)
-        }
-    } else if exp == 0x1f {
-        sign | 0x7f80_0000 | (mant << 13) // inf / NaN
-    } else {
-        sign | ((exp + (127 - 15)) << 23) | (mant << 13)
-    };
-    f32::from_bits(bits)
-}
+use crate::float16::f16_bits_to_f32;
 
 fn read_f32_bytes(bytes: &[u8], order: &DatatypeByteOrder) -> f32 {
     let mut buf = [0u8; 4];
@@ -1680,20 +1883,6 @@ fn effective_bits(size: usize, bit_offset: u16, bit_precision: u16) -> (u32, u32
     (bit_offset as u32, prec)
 }
 
-/// `(bit_offset, bit_precision)` for a fixed-point datatype, full width for
-/// other types.
-fn fixed_bits(datatype: &Datatype) -> (u32, u32) {
-    match datatype {
-        Datatype::FixedPoint {
-            size,
-            bit_offset,
-            bit_precision,
-            ..
-        } => effective_bits(*size as usize, *bit_offset, *bit_precision),
-        _ => (0, 0),
-    }
-}
-
 /// Whether a datatype occupies its full storage width (bit offset 0, precision
 /// == size·8), in which case the bulk-copy fast read paths apply. Non
 /// fixed-point types are treated as full width.
@@ -1906,6 +2095,67 @@ mod tests {
         assert_eq!(read_as_u64(&raw, &dt).unwrap(), vec![4095, 1, 2048]);
     }
 
+    #[test]
+    fn float_to_int_truncates_and_saturates() {
+        // Values libhdf5 hands to an undefined C cast: NaN reads as 0 and
+        // exactly 2^63 saturates instead of wrapping to i64::MIN.
+        let dt = make_f64_le_type();
+        let vals = [f64::NAN, 2f64.powi(63), -2.5, 2.0f64.powi(64)];
+        let raw: Vec<u8> = vals.iter().flat_map(|v| v.to_le_bytes()).collect();
+        assert_eq!(
+            read_as_i64(&raw, &dt).unwrap(),
+            vec![0, i64::MAX, -2, i64::MAX]
+        );
+        assert_eq!(
+            read_as_u64(&raw, &dt).unwrap(),
+            vec![0, 1 << 63, 0, u64::MAX]
+        );
+        assert_eq!(
+            read_as_i32(&raw, &dt).unwrap(),
+            vec![0, i32::MAX, -2, i32::MAX]
+        );
+    }
+
+    #[test]
+    fn bfloat16_and_fp8_decode_by_fields() {
+        // bfloat16 is a 2-byte float that is not IEEE half.
+        let bf16 = Datatype::FloatingPoint {
+            size: 2,
+            byte_order: DatatypeByteOrder::LittleEndian,
+            bit_offset: 0,
+            bit_precision: 16,
+            exponent_location: 7,
+            exponent_size: 8,
+            mantissa_location: 0,
+            mantissa_size: 7,
+            exponent_bias: 127,
+        };
+        let raw: Vec<u8> = [0x3FC0u16, 0xC010, 0x7F80, 0x0001]
+            .iter()
+            .flat_map(|v| v.to_le_bytes())
+            .collect();
+        let got = read_as_f64(&raw, &bf16).unwrap();
+        assert_eq!(&got[..3], &[1.5, -2.25, f64::INFINITY]);
+        assert_eq!(got[3], 2f64.powi(-133)); // smallest subnormal
+        assert_eq!(read_as_f32(&raw, &bf16).unwrap()[..2], [1.5, -2.25]);
+
+        // FP8 E4M3: 1, -1, 2, 0, NaN (IEEE-style, as libhdf5 treats it).
+        let e4m3 = Datatype::FloatingPoint {
+            size: 1,
+            byte_order: DatatypeByteOrder::LittleEndian,
+            bit_offset: 0,
+            bit_precision: 8,
+            exponent_location: 3,
+            exponent_size: 4,
+            mantissa_location: 0,
+            mantissa_size: 3,
+            exponent_bias: 7,
+        };
+        let got = read_as_f64(&[0x38, 0xB8, 0x40, 0x00, 0x7E], &e4m3).unwrap();
+        assert_eq!(&got[..4], &[1.0, -1.0, 2.0, 0.0]);
+        assert!(got[4].is_nan());
+    }
+
     #[test]
     fn full_width_signed_unchanged() {
         // Regression: full-width 32-bit signed must be unaffected.

crates/clawhdf5-format/src/datatype.rs

@@ -4,7 +4,7 @@
 //! for compound, enumeration, variable-length, and array types.
 
 #[cfg(not(feature = "std"))]
-use alloc::{boxed::Box, string::String, vec, vec::Vec};
+use alloc::{boxed::Box, format, string::String, vec, vec::Vec};
 
 use byteorder::{ByteOrder, LittleEndian};
 
@@ -137,6 +137,17 @@ pub enum Datatype {
     },
 }
 
+/// Longest opaque tag that can be stored: its NUL-padded length must fit
+/// the 8-bit length in the datatype's class bits.
+pub const MAX_OPAQUE_TAG_LEN: usize = 248;
+
+/// An opaque tag up to (not including) its first NUL.
+fn opaque_tag_text(tag: &[u8]) -> &[u8] {
+    tag.iter()
+        .position(|&b| b == 0)
+        .map_or(tag, |end| &tag[..end])
+}
+
 fn ensure_len(data: &[u8], offset: usize, needed: usize) -> Result<(), FormatError> {
     match offset.checked_add(needed) {
         Some(end) if end <= data.len() => Ok(()),
@@ -361,7 +372,10 @@ impl Datatype {
                 // Opaque
                 let tag_len = bf0 as usize;
                 ensure_len(data, pos, tag_len)?;
-                let tag = data[pos..pos + tag_len].to_vec();
+                // The stored tag is NUL-padded to a multiple of 8 bytes; the
+                // tag itself ends at the first NUL (libhdf5 reads it with
+                // `strndup`).
+                let tag = opaque_tag_text(&data[pos..pos + tag_len]).to_vec();
                 // Tags are padded to multiple of 8 bytes
                 let padded = (tag_len + 7) & !7;
                 let pos = 8 + padded; // from start of properties
@@ -640,7 +654,8 @@ impl Datatype {
                 mantissa_size,
                 exponent_bias,
             } => {
-                let mut bf0 = 0x20u8; // bit 5: sign location bit (standard IEEE 754)
+                // Bits 4-5: mantissa normalization = 2 (implied leading 1, IEEE 754).
+                let mut bf0 = 0x20u8;
                 match byte_order {
                     DatatypeByteOrder::BigEndian => {
                         bf0 |= 0x01;
@@ -650,9 +665,14 @@ impl Datatype {
                     }
                     _ => {}
                 }
-                // bf[1] bits 0-1: mantissa normalization = 2 (MSB not stored, IEEE 754)
-                let bf1 = 0x3fu8; // matching what h5py generates
-                let mut buf = Self::build_header(1, 1, [bf0, bf1, 0], *size);
+                // Bits 8-15: the sign bit's position, the top bit of the value.
+                // This was hard-coded to 63, which is right only for f64: the
+                // HDF5 library rejects any other float with "sign bit position
+                // out of bounds", so every f32 dataset and attribute we wrote
+                // was unreadable by h5py and libhdf5.
+                let sign_location =
+                    (u32::from(*bit_offset) + u32::from(*bit_precision)).saturating_sub(1) as u8;
+                let mut buf = Self::build_header(1, 1, [bf0, sign_location, 0], *size);
                 buf.extend_from_slice(&bit_offset.to_le_bytes());
                 buf.extend_from_slice(&bit_precision.to_le_bytes());
                 buf.push(*exponent_location);
@@ -761,7 +781,77 @@ impl Datatype {
                 buf.extend_from_slice(&base_type.serialize());
                 buf
             }
-            _ => Vec::new(),
+            Datatype::Time {
+                size,
+                bit_precision,
+            } => {
+                // Byte order is not modelled for time types; write little-endian.
+                let mut buf = Self::build_header(2, 1, [0, 0, 0], *size);
+                buf.extend_from_slice(&bit_precision.to_le_bytes());
+                buf
+            }
+            Datatype::BitField {
+                size,
+                byte_order,
+                bit_offset,
+                bit_precision,
+            } => {
+                let bf0 = u8::from(matches!(byte_order, DatatypeByteOrder::BigEndian));
+                let mut buf = Self::build_header(4, 1, [bf0, 0, 0], *size);
+                buf.extend_from_slice(&bit_offset.to_le_bytes());
+                buf.extend_from_slice(&bit_precision.to_le_bytes());
+                buf
+            }
+            Datatype::Opaque { size, tag } => {
+                // The tag is stored NUL-padded to a multiple of 8 bytes and the
+                // padded length goes in the class bits, as libhdf5 writes it.
+                // A tag longer than MAX_OPAQUE_TAG_LEN cannot be encoded;
+                // `check_encodable` rejects it before a file is written.
+                let tag = opaque_tag_text(tag);
+                let tag = &tag[..tag.len().min(MAX_OPAQUE_TAG_LEN)];
+                let padded = tag.len().div_ceil(8) * 8;
+                let mut buf = Self::build_header(5, 1, [padded as u8, 0, 0], *size);
+                buf.extend_from_slice(tag);
+                buf.resize(8 + padded, 0);
+                buf
+            }
+            Datatype::Reference { size, ref_type } => {
+                // Legacy references are datatype version 1; the H5T_STD_REF
+                // kinds only exist from version 4, which also carries their
+                // encoding version (1) in the high nibble.
+                let (version, bf0) = match ref_type {
+                    ReferenceType::Object => (1, 0),
+                    ReferenceType::DatasetRegion => (1, 1),
+                    ReferenceType::Object2 => (4, 0x12),
+                    ReferenceType::DatasetRegion2 => (4, 0x13),
+                    ReferenceType::Attribute => (4, 0x14),
+                };
+                Self::build_header(7, version, [bf0, 0, 0], *size)
+            }
+        }
+    }
+
+    /// Check that this datatype can be written: every part of it has an
+    /// on-disk encoding. [`Self::serialize`] cannot report errors, so the
+    /// writer calls this first.
+    pub fn check_encodable(&self) -> Result<(), FormatError> {
+        match self {
+            Datatype::Opaque { tag, .. } if opaque_tag_text(tag).len() > MAX_OPAQUE_TAG_LEN => {
+                Err(FormatError::SerializationError(format!(
+                    "opaque tag is {} bytes; at most {MAX_OPAQUE_TAG_LEN} can be stored",
+                    opaque_tag_text(tag).len()
+                )))
+            }
+            Datatype::String { size: 0, .. } => Err(FormatError::SerializationError(
+                "fixed-length string datatype of size 0 (libhdf5 requires at least 1 byte)".into(),
+            )),
+            Datatype::Compound { members, .. } => members
+                .iter()
+                .try_for_each(|m| m.datatype.check_encodable()),
+            Datatype::Enumeration { base_type, .. }
+            | Datatype::VariableLength { base_type, .. }
+            | Datatype::Array { base_type, .. } => base_type.check_encodable(),
+            _ => Ok(()),
         }
     }
 
@@ -818,6 +908,24 @@ fn build_dt_header(class: u8, version: u8, bf: [u8; 3], size: u32) -> Vec<u8> {
 mod tests {
     use super::*;
 
+    #[test]
+    fn float_sign_location_is_the_top_bit_of_the_value() {
+        // The HDF5 library rejects a float whose sign position is not inside
+        // its precision; this was hard-coded to 63, so every f32 we wrote was
+        // unreadable by h5py. Byte 2 of the message is the sign position.
+        use crate::type_builders::{make_f16_type, make_f32_type, make_f64_type};
+        for (dt, sign) in [
+            (make_f16_type(), 15),
+            (make_f32_type(), 31),
+            (make_f64_type(), 63),
+        ] {
+            let bytes = dt.serialize();
+            assert_eq!(bytes[2], sign, "{dt:?}");
+            let (parsed, _) = Datatype::parse(&bytes).unwrap();
+            assert_eq!(parsed, dt);
+        }
+    }
+
     // Helper to build a fixed-point datatype message
     fn build_fixed_point(
         size: u32,
@@ -1601,6 +1709,122 @@ mod tests {
         );
     }
 
+    fn hex(s: &str) -> Vec<u8> {
+        (0..s.len())
+            .step_by(2)
+            .map(|i| u8::from_str_radix(&s[i..i + 2], 16).unwrap())
+            .collect()
+    }
+
+    /// `serialize` used to return an empty message for these four classes,
+    /// which libhdf5 rejects ("ran off end of input buffer while decoding").
+    /// Expected bytes are libhdf5's own encoding (HDF5 2.0 `H5Tencode`, or the
+    /// datatype message of an HDF5 2.0 file for `H5T_STD_REF`).
+    #[test]
+    fn serialize_matches_libhdf5_for_time_bitfield_opaque_reference() {
+        let cases = [
+            (
+                Datatype::Reference {
+                    size: 8,
+                    ref_type: ReferenceType::Object,
+                },
+                "1700000008000000",
+            ),
+            (
+                Datatype::Reference {
+                    size: 12,
+                    ref_type: ReferenceType::DatasetRegion,
+                },
+                "170100000c000000",
+            ),
+            (
+                Datatype::Reference {
+                    size: 18,
+                    ref_type: ReferenceType::Object2,
+                },
+                "4712000012000000",
+            ),
+            (
+                Datatype::BitField {
+                    size: 1,
+                    byte_order: DatatypeByteOrder::LittleEndian,
+                    bit_offset: 0,
+                    bit_precision: 8,
+                },
+                "140000000100000000000800",
+            ),
+            (
+                Datatype::BitField {
+                    size: 2,
+                    byte_order: DatatypeByteOrder::BigEndian,
+                    bit_offset: 0,
+                    bit_precision: 16,
+                },
+                "140100000200000000001000",
+            ),
+            (
+                Datatype::Opaque {
+                    size: 4,
+                    tag: b"mytag".to_vec(),
+                },
+                "15080000040000006d79746167000000",
+            ),
+            (
+                Datatype::Opaque {
+                    size: 4,
+                    tag: b"12345678".to_vec(),
+                },
+                "15080000040000003132333435363738",
+            ),
+            (
+                Datatype::Opaque {
+                    size: 4,
+                    tag: vec![],
+                },
+                "1500000004000000",
+            ),
+            (
+                Datatype::Time {
+                    size: 4,
+                    bit_precision: 32,
+                },
+                "12000000040000002000",
+            ),
+        ];
+        for (dt, expected) in cases {
+            let bytes = dt.serialize();
+            assert_eq!(bytes, hex(expected), "{dt:?}");
+            let (parsed, consumed) = Datatype::parse(&bytes).unwrap();
+            assert_eq!(parsed, dt);
+            assert_eq!(consumed, bytes.len());
+        }
+    }
+
+    #[test]
+    fn opaque_tag_padding_is_not_part_of_the_tag() {
+        // libhdf5 pads "mytag" to 8 bytes; parsing must not return the NULs,
+        // or copying the type would grow the tag.
+        let (dt, _) = Datatype::parse(&hex("15080000040000006d79746167000000")).unwrap();
+        assert_eq!(
+            dt,
+            Datatype::Opaque {
+                size: 4,
+                tag: b"mytag".to_vec()
+            }
+        );
+        let long = Datatype::Opaque {
+            size: 1,
+            tag: vec![b'x'; MAX_OPAQUE_TAG_LEN + 1],
+        };
+        assert!(long.check_encodable().is_err());
+        let ok = Datatype::Opaque {
+            size: 1,
+            tag: vec![b'x'; MAX_OPAQUE_TAG_LEN],
+        };
+        assert!(ok.check_encodable().is_ok());
+        assert_eq!(Datatype::parse(&ok.serialize()).unwrap().0, ok);
+    }
+
     #[test]
     fn test_error_invalid_reference_type() {
         let buf = build_dt_header(7, 1, [5, 0, 0], 8);

crates/clawhdf5-format/src/ea_writer.rs

@@ -7,7 +7,7 @@ extern crate alloc;
 use alloc::{vec, vec::Vec};
 
 use crate::checksum::jenkins_lookup3;
-use crate::chunked_write::WrittenChunk;
+use crate::chunked_write::{WrittenChunk, filtered_chunk_size_len, push_addr, push_index_element};
 
 /// Serialize a v4 Extensible Array layout message.
 pub(crate) fn serialize_v4_extensible_array(
@@ -58,11 +58,11 @@ pub(crate) fn serialize_v4_extensible_array(
     buf.push(4);
 
     // EA creation parameters (must match AEHD and HDF5 C library defaults)
-    buf.push(32); // max_nelmts_bits
-    buf.push(4); // idx_blk_elmts
-    buf.push(4); // super_blk_min_data_ptrs
-    buf.push(16); // data_blk_min_elmts
-    buf.push(10); // max_dblk_page_nelmts_bits
+    buf.push(MAX_NELMTS_BITS);
+    buf.push(IDX_BLK_ELMTS);
+    buf.push(SUP_BLK_MIN_DATA_PTRS);
+    buf.push(DATA_BLK_MIN_ELMTS);
+    buf.push(MAX_DBLK_PAGE_NELMTS_BITS);
 
     // EA header address
     match offset_size {
@@ -74,304 +74,281 @@ pub(crate) fn serialize_v4_extensible_array(
     buf
 }
 
+// EA creation parameters — the HDF5 library's defaults for chunk indexes
+// (`H5D_EARRAY_*`); the layout message above and the header must agree.
+const MAX_NELMTS_BITS: u8 = 32;
+const IDX_BLK_ELMTS: u8 = 4;
+const SUP_BLK_MIN_DATA_PTRS: u8 = 4;
+const DATA_BLK_MIN_ELMTS: u8 = 16;
+const MAX_DBLK_PAGE_NELMTS_BITS: u8 = 10;
+
+/// One data block of the array: its first element (relative to the end of
+/// the index block's own elements), element count, and address when it is
+/// allocated.
+struct DataBlock {
+    start: usize,
+    nelmts: usize,
+    addr: Option<u64>,
+}
+
 /// Build a complete Extensible Array at a known absolute address.
 ///
-/// For simplicity, we put all elements inline in the index block when the
-/// number of chunks is small (up to idx_blk_elmts), otherwise use inline +
-/// direct data blocks.
+/// `slots[i]` is the element at linear index `i` (see `chunk_grid`); `None`
+/// marks an unallocated chunk. The first `IDX_BLK_ELMTS` elements live in
+/// the index block, the rest in data blocks grouped by super block level
+/// exactly as `H5EA__hdr_init` sizes them: level `u` has `2^(u/2)` data
+/// blocks of `DATA_BLK_MIN_ELMTS * 2^ceil(u/2)` elements. The data blocks of
+/// the first levels are addressed straight from the index block; later
+/// levels go through a super block (EASB). Data blocks larger than a page
+/// (`2^MAX_DBLK_PAGE_NELMTS_BITS` elements) are paged, with their page-init
+/// bits kept in the owning super block. Only blocks holding a defined element
+/// are allocated; the rest keep the undefined address, as in a file the
+/// library wrote.
 pub fn build_extensible_array_at(
-    chunks: &[WrittenChunk],
+    slots: &[Option<WrittenChunk>],
     offset_size: u8,
     length_size: u8,
     has_filters: bool,
     ea_base_address: u64,
 ) -> Vec<u8> {
     let os = offset_size as usize;
-    let num_elements = chunks.len();
-
-    // Compute element encoding size (same logic as Fixed Array)
-    let chunk_size_bytes: usize = if has_filters {
-        let max_raw = chunks.iter().map(|c| c.raw_size).max().unwrap_or(1);
-        let log2_val = if max_raw <= 1 {
-            0
-        } else {
-            63 - max_raw.leading_zeros()
-        };
-        let len = 1 + ((log2_val + 8) / 8) as usize;
-        len.min(8)
-    } else {
-        0
-    };
-
-    let elem_size = if has_filters {
-        os + chunk_size_bytes + 4
-    } else {
-        os
-    };
-
+    let chunk_size_bytes = has_filters.then(|| filtered_chunk_size_len(slots));
+    let elem_size = os + chunk_size_bytes.map_or(0, |n| n + 4);
     let client_id: u8 = if has_filters { 1 } else { 0 };
+    let arr_off_size = (MAX_NELMTS_BITS as usize).div_ceil(8);
+    let page_nelmts = 1usize << MAX_DBLK_PAGE_NELMTS_BITS;
+    let idx_blk = IDX_BLK_ELMTS as usize;
 
-    // EA creation parameters — must match HDF5 C library defaults exactly
-    let max_nelmts_bits: u8 = 32;
-    let idx_blk_elmts: u8 = 4;
-    let min_dblk_nelmts: u8 = 16;
-    let super_blk_min_nelmts: u8 = 4;
-    let max_dblk_nelmts_bits: u8 = 10;
+    // Elements past the last defined one are never realised
+    // (`max_idx_set` is one past the highest index ever set).
+    let max_idx_set = slots.iter().rposition(Option::is_some).map_or(0, |i| i + 1);
+    let slots = &slots[..max_idx_set];
+    let defined_in = |start: usize, n: usize| -> bool {
+        let lo = idx_blk.saturating_add(start).min(slots.len());
+        let hi = idx_blk
+            .saturating_add(start)
+            .saturating_add(n)
+            .min(slots.len());
+        slots[lo..hi].iter().any(Option::is_some)
+    };
 
-    // EAHD size: fixed(12) + 6 stats(6*length_size) + addr(offset_size) + checksum(4)
+    // Super block levels: (ndblks, dblk_nelmts, first element).
+    let log2_dmin = (DATA_BLK_MIN_ELMTS as u32).trailing_zeros() as usize;
+    let nsblks = 1 + MAX_NELMTS_BITS as usize - log2_dmin;
+    let ndblk_addrs = 2 * (SUP_BLK_MIN_DATA_PTRS as usize - 1);
+    let mut levels: Vec<(usize, usize, usize)> = Vec::with_capacity(nsblks);
+    let mut start = 0usize;
+    for u in 0..nsblks {
+        let ndblks = 1usize << (u / 2);
+        let nelmts = (DATA_BLK_MIN_ELMTS as usize) << u.div_ceil(2);
+        levels.push((ndblks, nelmts, start));
+        // Saturate: on 32-bit targets the last levels only need to compare
+        // as "beyond the end".
+        start = start.saturating_add(ndblks.saturating_mul(nelmts));
+    }
+    // Levels whose data blocks the index block addresses directly.
+    let mut direct_levels = 0;
+    let mut n = 0;
+    while n < ndblk_addrs {
+        n += levels[direct_levels].0;
+        direct_levels += 1;
+    }
+    let nsblk_addrs = nsblks - direct_levels;
+
+    let dblk_size = |nelmts: usize| -> usize {
+        let prefix = 4 + 1 + 1 + os + arr_off_size + 4;
+        if nelmts > page_nelmts {
+            prefix + (nelmts / page_nelmts) * (page_nelmts * elem_size + 4)
+        } else {
+            prefix + nelmts * elem_size
+        }
+    };
+    let sblk_bitmap_len = |ndblks: usize, nelmts: usize| -> usize {
+        if nelmts > page_nelmts {
+            ndblks * (nelmts / page_nelmts).div_ceil(8)
+        } else {
+            0
+        }
+    };
+
+    // Plan addresses: header, index block, the direct data blocks, then each
+    // allocated super block followed by its allocated data blocks.
     let aehd_size = 4 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 6 * length_size as usize + os + 4;
     let aeib_address = ea_base_address + aehd_size as u64;
+    let aeib_size = 4 + 1 + 1 + os + idx_blk * elem_size + ndblk_addrs * os + nsblk_addrs * os + 4;
+    let mut cursor = aeib_address + aeib_size as u64;
 
-    // Determine how many elements go inline vs data blocks
-    let n_inline = (idx_blk_elmts as usize).min(num_elements);
-    let remaining_after_inline = num_elements.saturating_sub(n_inline);
+    let mut ndata_blks = 0u64;
+    let mut data_blk_size = 0u64;
+    let mut nsuper_blks = 0u64;
+    let mut super_blk_size = 0u64;
+    let mut realized = idx_blk as u64;
 
-    // Compute super block layout per HDF5 spec
-    let sblk_min = super_blk_min_nelmts as usize;
-    let log2_dblk_min = if min_dblk_nelmts <= 1 {
-        0
-    } else {
-        (min_dblk_nelmts as u32).trailing_zeros() as usize
-    };
-    let nsblks = (max_nelmts_bits as usize).saturating_sub(log2_dblk_min) + 1;
-
-    // Direct data block addresses (from super blocks 0..sblk_min-1)
-    let mut dblk_sizes: Vec<usize> = Vec::new();
-    for sblk_idx in 0..sblk_min.min(nsblks) {
-        let ndblks = 1usize << (sblk_idx / 2);
-        let dblk_nelmts = (min_dblk_nelmts as usize) * (1 << sblk_idx.div_ceil(2));
-        for _ in 0..ndblks {
-            dblk_sizes.push(dblk_nelmts);
+    let mut plan_dblk = |cursor: &mut u64, start: usize, nelmts: usize| -> DataBlock {
+        let addr = defined_in(start, nelmts).then(|| {
+            let a = *cursor;
+            let size = dblk_size(nelmts) as u64;
+            *cursor += size;
+            ndata_blks += 1;
+            data_blk_size += size;
+            realized += nelmts as u64;
+            a
+        });
+        DataBlock {
+            start,
+            nelmts,
+            addr,
         }
-    }
-    let n_direct_dblks = dblk_sizes.len();
-
-    // Super block addresses (for super blocks sblk_min..nsblks-1)
-    let n_sblk_addrs = nsblks.saturating_sub(sblk_min);
-
-    // EAIB size
-    let aeib_size = 4
-        + 1
-        + 1
-        + os
-        + idx_blk_elmts as usize * elem_size
-        + n_direct_dblks * os
-        + n_sblk_addrs * os
-        + 4;
-
-    // Build AEHD
-    let mut aehd = Vec::with_capacity(aehd_size);
-    aehd.extend_from_slice(b"EAHD");
-    aehd.push(0); // version
-    aehd.push(client_id);
-    aehd.push(elem_size as u8);
-    aehd.push(max_nelmts_bits);
-    aehd.push(idx_blk_elmts);
-    aehd.push(min_dblk_nelmts);
-    aehd.push(super_blk_min_nelmts);
-    aehd.push(max_dblk_nelmts_bits);
-
-    // Count data blocks that will have chunks
-    let n_active_dblks: u64 = if remaining_after_inline > 0 {
-        let mut count = 0u64;
-        let mut ci = n_inline;
-        for &sz in &dblk_sizes {
-            if ci < num_elements {
-                count += 1;
-                ci += sz;
-            }
-        }
-        count
-    } else {
-        0
-    };
-    let blk_off_size = (max_nelmts_bits as usize).div_ceil(8);
-    let aedb_header_overhead = 4 + 1 + 1 + os + blk_off_size + 4;
-    let data_blk_total_size: u64 = if remaining_after_inline > 0 {
-        let mut total = 0u64;
-        let mut ci = n_inline;
-        for &sz in &dblk_sizes {
-            if ci < num_elements {
-                total += (aedb_header_overhead + sz * elem_size) as u64;
-                ci += sz;
-            }
-        }
-        total
-    } else {
-        0
-    };
-    let max_idx_set: u64 = if remaining_after_inline > 0 {
-        let mut max_set = idx_blk_elmts as u64;
-        let mut ci = n_inline;
-        for &sz in &dblk_sizes {
-            if ci < num_elements {
-                max_set += sz as u64;
-                ci += sz;
-            }
-        }
-        max_set
-    } else {
-        idx_blk_elmts as u64
     };
 
+    let mut direct: Vec<DataBlock> = Vec::with_capacity(ndblk_addrs);
+    for &(ndblks, nelmts, first) in &levels[..direct_levels] {
+        for k in 0..ndblks {
+            direct.push(plan_dblk(&mut cursor, first + k * nelmts, nelmts));
+        }
+    }
+    // (super block address, level, its data blocks)
+    let mut supers: Vec<(Option<u64>, usize, Vec<DataBlock>)> = Vec::with_capacity(nsblk_addrs);
+    for (u, &(ndblks, nelmts, first)) in levels.iter().enumerate().skip(direct_levels) {
+        if !defined_in(first, ndblks.saturating_mul(nelmts)) {
+            supers.push((None, u, Vec::new()));
+            continue;
+        }
+        let sb_size =
+            4 + 1 + 1 + os + arr_off_size + sblk_bitmap_len(ndblks, nelmts) + ndblks * os + 4;
+        let sb_addr = cursor;
+        cursor += sb_size as u64;
+        nsuper_blks += 1;
+        super_blk_size += sb_size as u64;
+        let dblks = (0..ndblks)
+            .map(|k| plan_dblk(&mut cursor, first + k * nelmts, nelmts))
+            .collect();
+        supers.push((Some(sb_addr), u, dblks));
+    }
+
+    let slot = |i: usize| slots.get(i).and_then(Option::as_ref);
     let write_length = |buf: &mut Vec<u8>, val: u64| match length_size {
         4 => buf.extend_from_slice(&(val as u32).to_le_bytes()),
         _ => buf.extend_from_slice(&val.to_le_bytes()),
     };
-    let write_addr = |buf: &mut Vec<u8>, val: u64| match offset_size {
-        4 => buf.extend_from_slice(&(val as u32).to_le_bytes()),
-        _ => buf.extend_from_slice(&val.to_le_bytes()),
+    let write_addr_opt = |buf: &mut Vec<u8>, addr: Option<u64>| match addr {
+        Some(a) => push_addr(buf, a, offset_size),
+        None => buf.extend(core::iter::repeat_n(0xFF, os)),
+    };
+    let block_prefix = |buf: &mut Vec<u8>, sig: &[u8; 4], block_off: usize| {
+        buf.extend_from_slice(sig);
+        buf.push(0); // version
+        buf.push(client_id);
+        push_addr(buf, ea_base_address, offset_size);
+        buf.extend_from_slice(&(block_off as u64).to_le_bytes()[..arr_off_size]);
+    };
+    // Serialise one data block (paged or not) onto `out`.
+    let write_dblk = |out: &mut Vec<u8>, db: &DataBlock| {
+        let at = out.len();
+        block_prefix(out, b"EADB", db.start);
+        let first = idx_blk + db.start;
+        if db.nelmts > page_nelmts {
+            // Paged: the prefix carries only its own checksum; each page
+            // follows with one of its own.
+            let sum = jenkins_lookup3(&out[at..]);
+            out.extend_from_slice(&sum.to_le_bytes());
+            for p in 0..db.nelmts / page_nelmts {
+                let page_at = out.len();
+                for e in 0..page_nelmts {
+                    let i = first + p * page_nelmts + e;
+                    push_index_element(out, slot(i), offset_size, chunk_size_bytes);
+                }
+                let sum = jenkins_lookup3(&out[page_at..]);
+                out.extend_from_slice(&sum.to_le_bytes());
+            }
+        } else {
+            for i in first..first + db.nelmts {
+                push_index_element(out, slot(i), offset_size, chunk_size_bytes);
+            }
+            let sum = jenkins_lookup3(&out[at..]);
+            out.extend_from_slice(&sum.to_le_bytes());
+        }
+        debug_assert_eq!(out.len() - at, dblk_size(db.nelmts));
     };
 
-    write_length(&mut aehd, 0);
-    write_length(&mut aehd, 0);
-    write_length(&mut aehd, n_active_dblks);
-    write_length(&mut aehd, data_blk_total_size);
-    write_length(&mut aehd, num_elements as u64);
-    write_length(&mut aehd, max_idx_set);
+    // Header (EAHD). The six statistics are, in order: super blocks, their
+    // bytes, data blocks, their bytes, max index set, elements realised.
+    let mut out = Vec::with_capacity((cursor - ea_base_address) as usize);
+    out.extend_from_slice(b"EAHD");
+    out.push(0); // version
+    out.push(client_id);
+    out.push(elem_size as u8);
+    out.push(MAX_NELMTS_BITS);
+    out.push(IDX_BLK_ELMTS);
+    out.push(DATA_BLK_MIN_ELMTS);
+    out.push(SUP_BLK_MIN_DATA_PTRS);
+    out.push(MAX_DBLK_PAGE_NELMTS_BITS);
+    write_length(&mut out, nsuper_blks);
+    write_length(&mut out, super_blk_size);
+    write_length(&mut out, ndata_blks);
+    write_length(&mut out, data_blk_size);
+    write_length(&mut out, max_idx_set as u64);
+    write_length(&mut out, realized);
+    push_addr(&mut out, aeib_address, offset_size);
+    let sum = jenkins_lookup3(&out);
+    out.extend_from_slice(&sum.to_le_bytes());
+    debug_assert_eq!(out.len(), aehd_size);
 
-    write_addr(&mut aehd, aeib_address);
-
-    let aehd_checksum = jenkins_lookup3(&aehd);
-    aehd.extend_from_slice(&aehd_checksum.to_le_bytes());
-    debug_assert_eq!(aehd.len(), aehd_size);
-
-    // Build AEIB
-    let mut aeib = Vec::with_capacity(aeib_size);
-    aeib.extend_from_slice(b"EAIB");
-    aeib.push(0);
-    aeib.push(client_id);
-
-    match offset_size {
-        4 => aeib.extend_from_slice(&(ea_base_address as u32).to_le_bytes()),
-        8 => aeib.extend_from_slice(&ea_base_address.to_le_bytes()),
-        _ => aeib.extend_from_slice(&ea_base_address.to_le_bytes()),
+    // Index block (EAIB): inline elements, data block and super block
+    // addresses.
+    let ib_start = out.len();
+    out.extend_from_slice(b"EAIB");
+    out.push(0);
+    out.push(client_id);
+    push_addr(&mut out, ea_base_address, offset_size);
+    for i in 0..idx_blk {
+        push_index_element(&mut out, slot(i), offset_size, chunk_size_bytes);
     }
-
-    // Inline elements
-    #[allow(clippy::needless_range_loop)]
-    for i in 0..idx_blk_elmts as usize {
-        if i < n_inline {
-            write_chunk_element(
-                &mut aeib,
-                &chunks[i],
-                offset_size,
-                has_filters,
-                chunk_size_bytes,
-            );
-        } else {
-            write_undefined_element(&mut aeib, offset_size, has_filters, chunk_size_bytes);
+    for db in &direct {
+        write_addr_opt(&mut out, db.addr);
     }
+    for (sb_addr, _, _) in &supers {
+        write_addr_opt(&mut out, *sb_addr);
     }
+    let sum = jenkins_lookup3(&out[ib_start..]);
+    out.extend_from_slice(&sum.to_le_bytes());
+    debug_assert_eq!(out.len() - ib_start, aeib_size);
 
-    // Data block addresses + build data blocks
-    let mut data_blocks_buf = Vec::new();
-    let dblks_base = aeib_address + aeib_size as u64;
-    let mut dblk_cursor = dblks_base;
-    let mut chunk_idx = n_inline;
-
-    for &nelmts in &dblk_sizes {
-        if chunk_idx >= num_elements {
-            match offset_size {
-                4 => aeib.extend_from_slice(&u32::MAX.to_le_bytes()),
-                8 => aeib.extend_from_slice(&u64::MAX.to_le_bytes()),
-                _ => aeib.extend_from_slice(&u64::MAX.to_le_bytes()),
+    for db in direct.iter().filter(|d| d.addr.is_some()) {
+        write_dblk(&mut out, db);
     }
+    for (sb_addr, u, dblks) in &supers {
+        if sb_addr.is_none() {
             continue;
         }
-
-        match offset_size {
-            4 => aeib.extend_from_slice(&(dblk_cursor as u32).to_le_bytes()),
-            8 => aeib.extend_from_slice(&dblk_cursor.to_le_bytes()),
-            _ => aeib.extend_from_slice(&dblk_cursor.to_le_bytes()),
-        }
-
-        // Build EADB
-        let mut aedb = Vec::new();
-        aedb.extend_from_slice(b"EADB");
-        aedb.push(0);
-        aedb.push(client_id);
-        match offset_size {
-            4 => aedb.extend_from_slice(&(ea_base_address as u32).to_le_bytes()),
-            8 => aedb.extend_from_slice(&ea_base_address.to_le_bytes()),
-            _ => aedb.extend_from_slice(&ea_base_address.to_le_bytes()),
-        }
-
-        let blk_off_size = (max_nelmts_bits as usize).div_ceil(8);
-        let blk_off_val = (chunk_idx - n_inline) as u64;
-        aedb.extend_from_slice(&blk_off_val.to_le_bytes()[..blk_off_size]);
-
-        for slot in 0..nelmts {
-            if chunk_idx + slot < num_elements {
-                write_chunk_element(
-                    &mut aedb,
-                    &chunks[chunk_idx + slot],
-                    offset_size,
-                    has_filters,
-                    chunk_size_bytes,
-                );
-            } else {
-                write_undefined_element(&mut aedb, offset_size, has_filters, chunk_size_bytes);
+        let (ndblks, nelmts, first) = levels[*u];
+        let sb_start = out.len();
+        block_prefix(&mut out, b"EASB", first);
+        if nelmts > page_nelmts {
+            // Page-init bits, `npages` per data block, packed MSB-first
+            // (`H5VM_bit_set`): every page of an allocated data block is
+            // written.
+            let npages = nelmts / page_nelmts;
+            let mut bitmap = vec![0u8; sblk_bitmap_len(ndblks, nelmts)];
+            for (k, db) in dblks.iter().enumerate() {
+                if db.addr.is_some() {
+                    for p in 0..npages {
+                        let bit = k * npages + p;
+                        bitmap[bit / 8] |= 0x80 >> (bit % 8);
                     }
                 }
-
-        let aedb_checksum = jenkins_lookup3(&aedb);
-        aedb.extend_from_slice(&aedb_checksum.to_le_bytes());
-
-        dblk_cursor += aedb.len() as u64;
-        data_blocks_buf.extend_from_slice(&aedb);
-        chunk_idx += nelmts;
             }
-
-    // Super block addresses (all undefined)
-    for _ in 0..n_sblk_addrs {
-        match offset_size {
-            4 => aeib.extend_from_slice(&u32::MAX.to_le_bytes()),
-            8 => aeib.extend_from_slice(&u64::MAX.to_le_bytes()),
-            _ => aeib.extend_from_slice(&u64::MAX.to_le_bytes()),
+            out.extend_from_slice(&bitmap);
+        }
+        for db in dblks {
+            write_addr_opt(&mut out, db.addr);
+        }
+        let sum = jenkins_lookup3(&out[sb_start..]);
+        out.extend_from_slice(&sum.to_le_bytes());
+        for db in dblks.iter().filter(|d| d.addr.is_some()) {
+            write_dblk(&mut out, db);
         }
     }
-
-    let aeib_checksum = jenkins_lookup3(&aeib);
-    aeib.extend_from_slice(&aeib_checksum.to_le_bytes());
-    debug_assert_eq!(aeib.len(), aeib_size);
-
-    let mut combined = aehd;
-    combined.extend_from_slice(&aeib);
-    combined.extend_from_slice(&data_blocks_buf);
-    combined
-}
-
-fn write_chunk_element(
-    buf: &mut Vec<u8>,
-    chunk: &WrittenChunk,
-    offset_size: u8,
-    has_filters: bool,
-    chunk_size_bytes: usize,
-) {
-    match offset_size {
-        4 => buf.extend_from_slice(&(chunk.address as u32).to_le_bytes()),
-        8 => buf.extend_from_slice(&chunk.address.to_le_bytes()),
-        _ => buf.extend_from_slice(&chunk.address.to_le_bytes()),
-    }
-    if has_filters {
-        let cs_bytes = chunk.compressed_size.to_le_bytes();
-        buf.extend_from_slice(&cs_bytes[..chunk_size_bytes]);
-        buf.extend_from_slice(&chunk.filter_mask.to_le_bytes());
-    }
-}
-
-fn write_undefined_element(
-    buf: &mut Vec<u8>,
-    offset_size: u8,
-    has_filters: bool,
-    chunk_size_bytes: usize,
-) {
-    let os = offset_size as usize;
-    // Use extend with repeat to avoid heap-allocating a temporary Vec on each call.
-    buf.extend(core::iter::repeat_n(0xFF, os));
-    if has_filters {
-        buf.extend(core::iter::repeat_n(0x00, chunk_size_bytes));
-        buf.extend_from_slice(&0u32.to_le_bytes());
-    }
+    debug_assert_eq!(out.len() as u64, cursor - ea_base_address);
+    out
 }

crates/clawhdf5-format/src/extensible_array.rs

@@ -9,6 +9,7 @@ extern crate alloc;
 #[cfg(not(feature = "std"))]
 use alloc::{format, vec, vec::Vec};
 
+use crate::chunk_grid::ChunkGrid;
 use crate::chunked_read::ChunkInfo;
 use crate::error::FormatError;
 
@@ -203,8 +204,7 @@ fn read_element(
     offset_size: u8,
     chunk_byte_size: u64,
     linear_index: usize,
-    num_chunks_per_dim: &[u64],
-    chunk_dimensions: &[u32],
+    grid: &ChunkGrid,
 ) -> Result<(Option<ChunkInfo>, usize), FormatError> {
     let os = offset_size as usize;
 
@@ -220,7 +220,10 @@ fn read_element(
             return Ok((None, os));
         }
         let address = read_offset(data, pos, offset_size)?;
-        let offsets = index_to_chunk_offsets(linear_index, num_chunks_per_dim, chunk_dimensions);
+        // A slot beyond the current extent is ignored, as the library does.
+        let Some(offsets) = grid.offsets(linear_index as u64) else {
+            return Ok((None, os));
+        };
         Ok((
             Some(ChunkInfo {
                 chunk_size: chunk_byte_size as u32,
@@ -261,7 +264,9 @@ fn read_element(
             data[fm_off + 2],
             data[fm_off + 3],
         ]);
-        let offsets = index_to_chunk_offsets(linear_index, num_chunks_per_dim, chunk_dimensions);
+        let Some(offsets) = grid.offsets(linear_index as u64) else {
+            return Ok((None, elem_total));
+        };
         Ok((
             Some(ChunkInfo {
                 chunk_size: chunk_size as u32,
@@ -274,27 +279,6 @@ fn read_element(
     }
 }
 
-/// Convert a linear chunk index to N-dimensional chunk offsets in dataset space.
-fn index_to_chunk_offsets(
-    index: usize,
-    num_chunks_per_dim: &[u64],
-    chunk_dimensions: &[u32],
-) -> Vec<u64> {
-    let rank = num_chunks_per_dim.len();
-    let mut offsets = vec![0u64; rank];
-    let mut remaining = index as u64;
-    for d in (0..rank).rev() {
-        let nchunks = num_chunks_per_dim[d];
-        if nchunks == 0 {
-            continue;
-        }
-        let chunk_idx = remaining % nchunks;
-        remaining /= nchunks;
-        offsets[d] = chunk_idx * chunk_dimensions[d] as u64;
-    }
-    offsets
-}
-
 /// Collect elements from a data block at the given offset.
 #[allow(clippy::too_many_arguments)]
 /// Layout of super block `u`, per the HDF5 spec: the number of data blocks it
@@ -339,8 +323,7 @@ fn read_data_block_elements(
     offset_size: u8,
     chunk_byte_size: u64,
     start_index: usize,
-    num_chunks_per_dim: &[u64],
-    chunk_dimensions: &[u32],
+    grid: &ChunkGrid,
     page_init: &[u8],
     first_page: usize,
 ) -> Result<Vec<ChunkInfo>, FormatError> {
@@ -376,8 +359,7 @@ fn read_data_block_elements(
                 offset_size,
                 chunk_byte_size,
                 first_index + i,
-                num_chunks_per_dim,
-                chunk_dimensions,
+                grid,
             )?;
             if let Some(ci) = info {
                 chunks.push(ci);
@@ -449,25 +431,19 @@ pub fn read_extensible_array_chunks(
     file_data: &[u8],
     header: &ExtensibleArrayHeader,
     dataset_dims: &[u64],
+    max_dims: Option<&[u64]>,
     chunk_dimensions: &[u32],
     element_size: u32,
     offset_size: u8,
     _length_size: u8,
 ) -> Result<Vec<ChunkInfo>, FormatError> {
-    let rank = chunk_dimensions.len();
     let os = offset_size as usize;
 
-    let mut num_chunks_per_dim = Vec::with_capacity(rank);
-    for d in 0..rank {
-        let ch_dim = chunk_dimensions[d] as u64;
-        if ch_dim == 0 {
-            return Err(FormatError::ChunkedReadError(
-                "chunk dimension is zero".into(),
-            ));
-        }
-        let ds_dim = dataset_dims[d];
-        num_chunks_per_dim.push(ds_dim.div_ceil(ch_dim));
-    }
+    // Linear indexes follow the maximum dimensions, with the unlimited
+    // dimension swizzled to the slowest position (see `chunk_grid`).
+    let dims_u64: Vec<u64> = chunk_dimensions.iter().map(|&d| d as u64).collect();
+    let grid = ChunkGrid::extensible_array(dataset_dims, max_dims, &dims_u64)?;
+    let grid = &grid;
 
     let chunk_byte_size: u64 =
         chunk_dimensions.iter().map(|&d| d as u64).product::<u64>() * element_size as u64;
@@ -557,8 +533,7 @@ pub fn read_extensible_array_chunks(
             offset_size,
             chunk_byte_size,
             i,
-            &num_chunks_per_dim,
-            chunk_dimensions,
+            grid,
         )?;
         if let Some(ci) = info {
             chunks.push(ci);
@@ -594,8 +569,7 @@ pub fn read_extensible_array_chunks(
                 offset_size,
                 chunk_byte_size,
                 global_index,
-                &num_chunks_per_dim,
-                chunk_dimensions,
+                grid,
                 &[],
                 0,
             )?);
@@ -625,8 +599,7 @@ pub fn read_extensible_array_chunks(
                 offset_size,
                 chunk_byte_size,
                 global_index,
-                &num_chunks_per_dim,
-                chunk_dimensions,
+                grid,
             )?);
         }
         global_index =
@@ -653,8 +626,7 @@ fn read_super_block(
     offset_size: u8,
     chunk_byte_size: u64,
     start_index: usize,
-    num_chunks_per_dim: &[u64],
-    chunk_dimensions: &[u32],
+    grid: &ChunkGrid,
 ) -> Result<Vec<ChunkInfo>, FormatError> {
     let os = offset_size as usize;
     let sb_header_size = 4 + 1 + 1 + os + arr_off_size(header);
@@ -710,8 +682,7 @@ fn read_super_block(
                 offset_size,
                 chunk_byte_size,
                 global_idx,
-                num_chunks_per_dim,
-                chunk_dimensions,
+                grid,
                 bitmap,
                 i * npages,
             )?);
@@ -735,35 +706,18 @@ mod tests {
     }
     #[test]
     fn index_to_offsets_1d() {
-        let num_chunks = vec![5u64];
-        let chunk_dims = vec![20u32];
-        assert_eq!(index_to_chunk_offsets(0, &num_chunks, &chunk_dims), vec![0]);
-        assert_eq!(
-            index_to_chunk_offsets(1, &num_chunks, &chunk_dims),
-            vec![20]
-        );
-        assert_eq!(
-            index_to_chunk_offsets(4, &num_chunks, &chunk_dims),
-            vec![80]
-        );
+        let g = ChunkGrid::fixed_array(&[100], None, &[20]).unwrap();
+        assert_eq!(g.offsets(0).unwrap(), vec![0]);
+        assert_eq!(g.offsets(1).unwrap(), vec![20]);
+        assert_eq!(g.offsets(4).unwrap(), vec![80]);
     }
 
     #[test]
     fn index_to_offsets_2d() {
-        let num_chunks = vec![3u64, 2];
-        let chunk_dims = vec![4u32, 3];
-        assert_eq!(
-            index_to_chunk_offsets(0, &num_chunks, &chunk_dims),
-            vec![0, 0]
-        );
-        assert_eq!(
-            index_to_chunk_offsets(1, &num_chunks, &chunk_dims),
-            vec![0, 3]
-        );
-        assert_eq!(
-            index_to_chunk_offsets(2, &num_chunks, &chunk_dims),
-            vec![4, 0]
-        );
+        let g = ChunkGrid::fixed_array(&[10, 6], None, &[4, 3]).unwrap();
+        assert_eq!(g.offsets(0).unwrap(), vec![0, 0]);
+        assert_eq!(g.offsets(1).unwrap(), vec![0, 3]);
+        assert_eq!(g.offsets(2).unwrap(), vec![4, 0]);
     }
 
     #[test]
@@ -830,7 +784,7 @@ mod tests {
             index_block_address: (usize::MAX - 4) as u64,
         };
         let buf = vec![0u8; 64];
-        let r = read_extensible_array_chunks(&buf, &header, &[100], &[20], 8, 8, 8);
+        let r = read_extensible_array_chunks(&buf, &header, &[100], None, &[20], 8, 8, 8);
         assert!(r.is_err());
     }
 
@@ -913,8 +867,16 @@ mod tests {
         let header = ExtensibleArrayHeader::parse(&file_data, aehd_offset, os, ls).unwrap();
         let ds_dims = vec![40u64]; // 2 chunks × 20 elements
         let chunk_dims = vec![20u32];
-        let chunks =
-            read_extensible_array_chunks(&file_data, &header, &ds_dims, &chunk_dims, 8, os, ls)
+        let chunks = read_extensible_array_chunks(
+            &file_data,
+            &header,
+            &ds_dims,
+            None,
+            &chunk_dims,
+            8,
+            os,
+            ls,
+        )
         .unwrap();
 
         assert_eq!(chunks.len(), 2);
@@ -1023,8 +985,16 @@ mod tests {
         let header = ExtensibleArrayHeader::parse(&file_data, aehd_offset, os, ls).unwrap();
         let ds_dims = vec![40u64];
         let chunk_dims = vec![10u32];
-        let chunks =
-            read_extensible_array_chunks(&file_data, &header, &ds_dims, &chunk_dims, 8, os, ls)
+        let chunks = read_extensible_array_chunks(
+            &file_data,
+            &header,
+            &ds_dims,
+            None,
+            &chunk_dims,
+            8,
+            os,
+            ls,
+        )
         .unwrap();
 
         assert_eq!(chunks.len(), 4);
@@ -1047,10 +1017,8 @@ mod tests {
     #[test]
     fn read_element_unallocated() {
         let data = vec![0xFFu8; 16];
-        let num_chunks = vec![5u64];
-        let chunk_dims = vec![10u32];
-        let (info, consumed) =
-            read_element(&data, 0, 0, 8, 8, 80, 0, &num_chunks, &chunk_dims).unwrap();
+        let grid = ChunkGrid::fixed_array(&[50], None, &[10]).unwrap();
+        let (info, consumed) = read_element(&data, 0, 0, 8, 8, 80, 0, &grid).unwrap();
         assert!(info.is_none());
         assert_eq!(consumed, 8);
     }
@@ -1069,20 +1037,9 @@ mod tests {
         // Filter mask
         data[12..16].copy_from_slice(&0u32.to_le_bytes());
 
-        let num_chunks = vec![5u64];
-        let chunk_dims = vec![10u32];
-        let (info, consumed) = read_element(
-            &data,
-            0,
-            1,
-            elem_size as u8,
-            os,
-            80,
-            2,
-            &num_chunks,
-            &chunk_dims,
-        )
-        .unwrap();
+        let grid = ChunkGrid::fixed_array(&[50], None, &[10]).unwrap();
+        let (info, consumed) =
+            read_element(&data, 0, 1, elem_size as u8, os, 80, 2, &grid).unwrap();
         let ci = info.unwrap();
         assert_eq!(ci.address, 0x2000);
         assert_eq!(ci.chunk_size, 120);

crates/clawhdf5-format/src/file_writer.rs

@@ -4,7 +4,7 @@
 //! link messages, contiguous datasets, inline and dense attributes.
 
 #[cfg(not(feature = "std"))]
-use alloc::{string::String, string::ToString, vec, vec::Vec};
+use alloc::{format, string::String, string::ToString, vec, vec::Vec};
 
 use crate::attribute::AttributeMessage;
 use crate::chunked_write::{
@@ -19,7 +19,7 @@ use crate::metadata_index::{DatasetMetadata, MetadataBlock, MetadataIndex};
 use crate::object_header_writer::ObjectHeaderWriter;
 use crate::superblock::Superblock;
 use crate::type_builders::{
-    DatasetBuilder, FillTime, FinishedGroup, GroupBuilder, build_attr_message,
+    DatasetBuilder, FinishedGroup, GroupBuilder, build_attr_message, fill_value_message,
 };
 
 // Re-export public types that moved to type_builders for API compatibility.
@@ -33,6 +33,49 @@ pub(crate) const OFFSET_SIZE: u8 = 8;
 pub(crate) const LENGTH_SIZE: u8 = 8;
 const SUPERBLOCK_SIZE: usize = 48;
 
+/// Largest raw data a compact dataset can hold: the layout message (version,
+/// class, 2-byte size, data) must fit an object header message, whose size
+/// field is 2 bytes. Bigger "compact" requests fall back to contiguous storage.
+const MAX_COMPACT_DATA_SIZE: usize = crate::object_header_writer::MAX_MESSAGE_SIZE - 4;
+
+/// libhdf5's bounds on a file space page size (`H5F_FILE_SPACE_PAGE_SIZE_MIN`
+/// and `_MAX`).
+const MIN_FILE_SPACE_PAGE_SIZE: u32 = 512;
+const MAX_FILE_SPACE_PAGE_SIZE: u32 = 1024 * 1024 * 1024;
+
+/// Superblock extension object header for a file using the paged file-space
+/// strategy: a single File Space Info message (0x0017), as libhdf5 writes it
+/// for `fs_strategy="page"` without persisted free space.
+fn build_paged_superblock_extension(page_size: u32) -> Result<Vec<u8>, FormatError> {
+    let mut fsinfo = Vec::new();
+    fsinfo.push(1); // version
+    fsinfo.push(1); // strategy: H5F_FSPACE_STRATEGY_PAGE
+    fsinfo.push(0); // persisting free space: no
+    write_length(&mut fsinfo, 1, LENGTH_SIZE); // free-space section threshold
+    write_length(&mut fsinfo, u64::from(page_size), LENGTH_SIZE);
+    fsinfo.extend_from_slice(&0u16.to_le_bytes()); // page end metadata threshold
+    write_undef_offset(&mut fsinfo, OFFSET_SIZE); // EOA before free-space info
+    let mut w = ObjectHeaderWriter::new();
+    // Flags as libhdf5 sets them: bit 2 (never share) and bit 4 (mark if
+    // unknown). Not constant: libhdf5 rewrites the message when it closes a
+    // file it opened for writing.
+    w.add_message_with_flags(MessageType::Unknown(0x0017), fsinfo, 0x14);
+    w.serialize()
+}
+
+/// A group or dataset name must be one path component: not empty, not ".",
+/// and without '/'. `FileWriter` writes a root group plus one level of
+/// groups, and cannot create intermediate groups for a path.
+fn check_link_name(name: &str) -> Result<(), FormatError> {
+    if name.is_empty() || name == "." || name.contains('/') {
+        return Err(FormatError::SerializationError(format!(
+            "invalid object name {name:?}: names must be a single path component \
+             (FileWriter does not create nested groups)"
+        )));
+    }
+    Ok(())
+}
+
 /// Threshold for switching from compact (inline) to dense attribute storage.
 const DENSE_ATTR_THRESHOLD: usize = 8;
 
@@ -50,12 +93,12 @@ pub(crate) fn build_chunked_dataset_oh(
     pipeline_message: Option<&[u8]>,
     attrs: &[AttributeMessage],
     dense_blob: Option<&DenseAttrBlob>,
-    fill_time: FillTime,
-) -> Vec<u8> {
+    fill_message: &[u8],
+) -> Result<Vec<u8>, FormatError> {
     let mut w = ObjectHeaderWriter::new();
     w.add_message_with_flags(MessageType::Datatype, dt.serialize(), 0x01);
     w.add_message(MessageType::Dataspace, ds.serialize(LENGTH_SIZE));
-    w.add_message_with_flags(MessageType::FillValue, vec![3, fill_time.to_byte()], 0x01);
+    w.add_message_with_flags(MessageType::FillValue, fill_message.to_vec(), 0x01);
     w.add_message(MessageType::DataLayout, layout_message.to_vec());
     if let Some(pm) = pipeline_message {
         w.add_message(MessageType::FilterPipeline, pm.to_vec());
@@ -77,15 +120,21 @@ pub(crate) fn build_dataset_oh(
     data_size: u64,
     attrs: &[AttributeMessage],
     dense_blob: Option<&DenseAttrBlob>,
-    fill_time: FillTime,
-) -> Vec<u8> {
+    fill_message: &[u8],
+) -> Result<Vec<u8>, FormatError> {
     let mut w = ObjectHeaderWriter::new();
     w.add_message_with_flags(MessageType::Datatype, dt.serialize(), 0x01);
     w.add_message(MessageType::Dataspace, ds.serialize(LENGTH_SIZE));
-    w.add_message_with_flags(MessageType::FillValue, vec![3, fill_time.to_byte()], 0x01);
+    w.add_message_with_flags(MessageType::FillValue, fill_message.to_vec(), 0x01);
     let mut dl = Vec::new();
     dl.push(4); // version
     dl.push(1); // class = contiguous
+    // An empty dataset has no storage: its address must be the undefined
+    // address, as libhdf5 writes it. A real address with size 0 trips
+    // libhdf5's `addr + size <= addr` overflow check, and it refuses the
+    // dataset as "invalid dataset size, likely file corruption" — which made
+    // every store with no sessions or knowledge graph unreadable by h5py.
+    let data_addr = if data_size == 0 { u64::MAX } else { data_addr };
     dl.extend_from_slice(&data_addr.to_le_bytes());
     dl.extend_from_slice(&data_size.to_le_bytes());
     w.add_message(MessageType::DataLayout, dl);
@@ -106,12 +155,12 @@ pub(crate) fn build_compact_dataset_oh(
     data: &[u8],
     attrs: &[AttributeMessage],
     dense_blob: Option<&DenseAttrBlob>,
-    fill_time: FillTime,
-) -> Vec<u8> {
+    fill_message: &[u8],
+) -> Result<Vec<u8>, FormatError> {
     let mut w = ObjectHeaderWriter::new();
     w.add_message_with_flags(MessageType::Datatype, dt.serialize(), 0x01);
     w.add_message(MessageType::Dataspace, ds.serialize(LENGTH_SIZE));
-    w.add_message_with_flags(MessageType::FillValue, vec![3, fill_time.to_byte()], 0x01);
+    w.add_message_with_flags(MessageType::FillValue, fill_message.to_vec(), 0x01);
     // Compact layout message: version=4, class=0, u16 size, inline data
     let mut dl = Vec::new();
     dl.push(4); // version
@@ -134,7 +183,7 @@ pub(crate) fn build_group_oh(
     dense_link_info: Option<&[u8]>,
     attrs: &[AttributeMessage],
     dense_blob: Option<&DenseAttrBlob>,
-) -> Vec<u8> {
+) -> Result<Vec<u8>, FormatError> {
     let mut w = ObjectHeaderWriter::new();
     if let Some(li) = dense_link_info {
         // Dense link storage: a LinkInfo pointing at the fractal heap + name
@@ -896,12 +945,12 @@ pub(crate) fn build_vds_dataset_oh(
     global_heap_addr: u64,
     attrs: &[AttributeMessage],
     dense_blob: Option<&DenseAttrBlob>,
-    fill_time: FillTime,
-) -> Vec<u8> {
+    fill_message: &[u8],
+) -> Result<Vec<u8>, FormatError> {
     let mut w = ObjectHeaderWriter::new();
     w.add_message_with_flags(MessageType::Datatype, dt.serialize(), 0x01);
     w.add_message(MessageType::Dataspace, ds.serialize(LENGTH_SIZE));
-    w.add_message_with_flags(MessageType::FillValue, vec![3, fill_time.to_byte()], 0x01);
+    w.add_message_with_flags(MessageType::FillValue, fill_message.to_vec(), 0x01);
     // VDS layout message: version=4, class=3, global_heap_address(8), global_heap_index=1(4)
     let mut dl = Vec::new();
     dl.push(4u8); // version
@@ -950,7 +999,9 @@ pub struct FileWriter {
     alignment_threshold: usize,
     /// Global alignment boundary in bytes (0 = disabled).
     alignment_bytes: usize,
-    /// Page size for page-buffer mode. When set, a v4 superblock is written.
+    /// File space page size. When set, the file uses libhdf5's paged
+    /// file-space strategy (a File Space Info message in the superblock
+    /// extension).
     page_size: Option<u32>,
 }
 
@@ -982,9 +1033,16 @@ impl FileWriter {
         self
     }
 
-    /// Enable page-buffer mode with the given page size. Writing this causes
-    /// the file to be written with a v4 superblock (page_size field) instead
-    /// of the default v3.
+    /// Write the file with libhdf5's *paged* file-space strategy and the given
+    /// page size, as `H5Pset_file_space_strategy(H5F_FSPACE_STRATEGY_PAGE)` +
+    /// `H5Pset_file_space_page_size` (h5py: `fs_strategy="page"`,
+    /// `fs_page_size=...`) do: a v3 superblock with an extension holding a
+    /// File Space Info message, and the file padded to a whole number of
+    /// pages. Readers with a page buffer can then fetch metadata page by page.
+    ///
+    /// `page_size` must be between 512 bytes and 1 GiB (libhdf5's limits);
+    /// [`Self::finish`] fails otherwise. This used to write a "version 4"
+    /// superblock, which does not exist and no HDF5 library can open.
     pub fn with_page_size(&mut self, page_size: u32) -> &mut Self {
         self.page_size = Some(page_size);
         self
@@ -1009,6 +1067,14 @@ impl FileWriter {
 
     pub fn finish(self) -> Result<Vec<u8>, FormatError> {
         let page_size = self.page_size;
+        if let Some(ps) = page_size
+            && !(MIN_FILE_SPACE_PAGE_SIZE..=MAX_FILE_SPACE_PAGE_SIZE).contains(&ps)
+        {
+            return Err(FormatError::SerializationError(format!(
+                "file space page size {ps} is outside libhdf5's \
+                 {MIN_FILE_SPACE_PAGE_SIZE}..={MAX_FILE_SPACE_PAGE_SIZE} bytes"
+            )));
+        }
         struct DsFlat {
             name: String,
             dt: Datatype,
@@ -1017,7 +1083,8 @@ impl FileWriter {
             attrs: Vec<AttributeMessage>,
             chunk_options: ChunkOptions,
             maxshape: Option<Vec<u64>>,
-            fill_time: FillTime,
+            /// Serialized Fill Value message.
+            fill_message: Vec<u8>,
             compact: bool,
             alignment: usize,
             /// VDS source mappings (set for Virtual datasets).
@@ -1067,6 +1134,7 @@ impl FileWriter {
                 };
                 attrs.extend(p.build_attrs(&raw));
             }
+            let fill_message = fill_value_message(db.fill_time, db.fill_value.as_deref(), &dt)?;
             Ok(DsFlat {
                 name: db.name,
                 dt,
@@ -1075,13 +1143,26 @@ impl FileWriter {
                 attrs,
                 chunk_options: db.chunk_options,
                 maxshape: db.maxshape,
-                fill_time: db.fill_time,
+                fill_message,
                 compact: db.compact,
                 alignment: db.alignment,
                 virtual_sources: db.virtual_sources,
             })
         };
 
+        // Every name becomes a single link in its parent group. The writer
+        // has no nested groups, so a path like "a/b" would be stored as one
+        // link literally named "a/b" — which no HDF5 reader can resolve.
+        let root_names = self.root_datasets.iter().map(|d| d.name.as_str());
+        let group_names = self.groups.iter().flat_map(|g| {
+            core::iter::once(g.name.as_str())
+                .chain(g.datasets.iter().map(|d| d.name.as_str()))
+                .chain(g.external_links.iter().map(|l| l.0.as_str()))
+        });
+        for name in root_names.chain(group_names) {
+            check_link_name(name)?;
+        }
+
         let mut all_ds: Vec<DsFlat> = Vec::new();
         let mut groups: Vec<GrpFlat> = Vec::new();
         let mut root_ds_indices: Vec<usize> = Vec::new();
@@ -1114,17 +1195,35 @@ impl FileWriter {
             root_attrs.push(build_attr_message(n, v));
         }
 
+        // Every datatype must have an on-disk encoding before anything is laid
+        // out: `Datatype::serialize` itself cannot report a failure.
+        let group_attrs = groups.iter().flat_map(|g| &g.attrs);
+        let ds_attrs = all_ds.iter().flat_map(|d| &d.attrs);
+        for a in root_attrs.iter().chain(group_attrs).chain(ds_attrs) {
+            a.datatype.check_encodable()?;
+        }
+        for d in &all_ds {
+            d.dt.check_encodable()?;
+        }
+
         let is_vds: Vec<bool> = all_ds.iter().map(|d| d.virtual_sources.is_some()).collect();
         let is_chunked: Vec<bool> = all_ds
             .iter()
             .enumerate()
-            .map(|(i, d)| !is_vds[i] && (d.chunk_options.is_chunked() || d.maxshape.is_some()))
+            .map(|(i, d)| {
+                // Only a dataset that can grow needs chunks; a maxshape equal
+                // to the shape is as fixed as no maxshape at all.
+                let resizable = d.maxshape.as_ref().is_some_and(|m| *m != d.ds.dimensions);
+                !is_vds[i] && (d.chunk_options.is_chunked() || resizable)
+            })
             .collect();
         // Determine which datasets use compact storage
         let is_compact: Vec<bool> = all_ds
             .iter()
             .enumerate()
-            .map(|(i, d)| !is_vds[i] && !is_chunked[i] && d.compact && d.raw.len() <= 65535)
+            .map(|(i, d)| {
+                !is_vds[i] && !is_chunked[i] && d.compact && d.raw.len() <= MAX_COMPACT_DATA_SIZE
+            })
             .collect();
         let root_dense = root_attrs.len() > DENSE_ATTR_THRESHOLD;
         let group_dense: Vec<bool> = groups
@@ -1163,9 +1262,9 @@ impl FileWriter {
                 }
                 let attr_blob = group_dense[gi].then(|| build_dense_attrs(&g.attrs, 0));
                 let dl = group_links_dense[gi].then_some(dummy_link_info.as_slice());
-                build_group_oh(&dummy_links, dl, &g.attrs, attr_blob.as_ref()).len()
+                build_group_oh(&dummy_links, dl, &g.attrs, attr_blob.as_ref()).map(|oh| oh.len())
             })
-            .collect();
+            .collect::<Result<_, _>>()?;
 
         let root_dummy_links: Vec<LinkMessage> = {
             let mut links = Vec::new();
@@ -1180,7 +1279,7 @@ impl FileWriter {
         let root_oh_size = {
             let attr_blob = root_dense.then(|| build_dense_attrs(&root_attrs, 0));
             let dl = root_links_dense.then_some(dummy_link_info.as_slice());
-            build_group_oh(&root_dummy_links, dl, &root_attrs, attr_blob.as_ref()).len()
+            build_group_oh(&root_dummy_links, dl, &root_attrs, attr_blob.as_ref())?.len()
         };
 
         struct DataBlob {
@@ -1208,8 +1307,8 @@ impl FileWriter {
                     0, // dummy address
                     &d.attrs,
                     dense_blob.as_ref(),
-                    d.fill_time,
-                );
+                    &d.fill_message,
+                )?;
                 // Global heap blob size is address-independent; compute it now
                 // so pass 2 can place it correctly.
                 let vds_mappings = d.virtual_sources.as_deref().unwrap_or(&[]);
@@ -1238,7 +1337,7 @@ impl FileWriter {
                     &pre,
                     dummy_cursor,
                     d.maxshape.as_deref(),
-                );
+                )?;
                 dummy_cursor += result.data_bytes.len() as u64;
                 let dense_blob = if ds_dense[i] {
                     Some(build_dense_attrs(&d.attrs, 0))
@@ -1252,8 +1351,8 @@ impl FileWriter {
                     result.pipeline_message.as_deref(),
                     &d.attrs,
                     dense_blob.as_ref(),
-                    d.fill_time,
-                );
+                    &d.fill_message,
+                )?;
                 dummy_blobs.push(DataBlob {
                     data: result.data_bytes,
                     oh_bytes: oh,
@@ -1271,8 +1370,8 @@ impl FileWriter {
                     &d.raw,
                     &d.attrs,
                     dense_blob.as_ref(),
-                    d.fill_time,
-                );
+                    &d.fill_message,
+                )?;
                 dummy_blobs.push(DataBlob {
                     data: vec![],
                     oh_bytes: oh,
@@ -1291,8 +1390,8 @@ impl FileWriter {
                     d.raw.len() as u64,
                     &d.attrs,
                     dense_blob.as_ref(),
-                    d.fill_time,
-                );
+                    &d.fill_message,
+                )?;
                 dummy_blobs.push(DataBlob {
                     data: d.raw.clone(),
                     oh_bytes: oh,
@@ -1304,12 +1403,12 @@ impl FileWriter {
         let actual_ds_oh_sizes: Vec<usize> = dummy_blobs.iter().map(|b| b.oh_bytes.len()).collect();
 
         // Pass 2: compute real addresses
-        // v4 superblocks add a 4-byte page_size field before the checksum.
-        let superblock_size = if page_size.is_some() {
-            SUPERBLOCK_SIZE + 4
-        } else {
-            SUPERBLOCK_SIZE
-        };
+        // A paged file carries its File Space Info in a superblock extension
+        // object header, placed right after the superblock.
+        let sb_ext = page_size
+            .map(build_paged_superblock_extension)
+            .transpose()?;
+        let superblock_size = SUPERBLOCK_SIZE + sb_ext.as_ref().map_or(0, Vec::len);
         let root_group_addr = superblock_size as u64;
         let mut cursor2 = superblock_size + root_oh_size;
 
@@ -1400,8 +1499,8 @@ impl FileWriter {
                     heap_addr,
                     &d.attrs,
                     ds_dense_blobs[i].as_ref(),
-                    d.fill_time,
-                );
+                    &d.fill_message,
+                )?;
                 ds_blobs2.push(DataBlob {
                     data: gcol_bytes.clone(),
                     oh_bytes: oh,
@@ -1418,7 +1517,7 @@ impl FileWriter {
                         .expect("chunked dataset missing precompressed cache"),
                     base_address,
                     d.maxshape.as_deref(),
-                );
+                )?;
                 cursor2 += result.data_bytes.len();
                 let oh = build_chunked_dataset_oh(
                     &d.dt,
@@ -1427,8 +1526,8 @@ impl FileWriter {
                     result.pipeline_message.as_deref(),
                     &d.attrs,
                     ds_dense_blobs[i].as_ref(),
-                    d.fill_time,
-                );
+                    &d.fill_message,
+                )?;
                 ds_blobs2.push(DataBlob {
                     data: result.data_bytes,
                     oh_bytes: oh,
@@ -1442,8 +1541,8 @@ impl FileWriter {
                     &d.raw,
                     &d.attrs,
                     ds_dense_blobs[i].as_ref(),
-                    d.fill_time,
-                );
+                    &d.fill_message,
+                )?;
                 ds_blobs2.push(DataBlob {
                     data: vec![],
                     oh_bytes: oh,
@@ -1467,8 +1566,8 @@ impl FileWriter {
                     d.raw.len() as u64,
                     &d.attrs,
                     ds_dense_blobs[i].as_ref(),
-                    d.fill_time,
-                );
+                    &d.fill_message,
+                )?;
                 let mut data = vec![0u8; padding];
                 data.extend_from_slice(&d.raw);
                 cursor2 += d.raw.len();
@@ -1483,11 +1582,16 @@ impl FileWriter {
         let actual_ds_oh_sizes2: Vec<usize> = ds_blobs2.iter().map(|b| b.oh_bytes.len()).collect();
         debug_assert_eq!(actual_ds_oh_sizes, actual_ds_oh_sizes2);
 
+        // libhdf5 ends a paged file on a page boundary.
+        let data_end = cursor2;
+        if let Some(ps) = page_size {
+            cursor2 = cursor2.next_multiple_of(ps as usize);
+        }
         let eof_addr2 = cursor2 as u64;
         let mut buf = Vec::with_capacity(cursor2);
 
         let sb = Superblock {
-            version: if page_size.is_some() { 4 } else { 3 },
+            version: 3,
             offset_size: OFFSET_SIZE,
             length_size: LENGTH_SIZE,
             base_address: 0,
@@ -1499,11 +1603,18 @@ impl FileWriter {
             free_space_address: None,
             driver_info_address: None,
             consistency_flags: 0,
-            superblock_extension_address: Some(u64::MAX),
+            superblock_extension_address: Some(if sb_ext.is_some() {
+                SUPERBLOCK_SIZE as u64
+            } else {
+                u64::MAX
+            }),
             checksum: None,
-            page_size,
+            page_size: None,
         };
         buf.extend_from_slice(&sb.serialize());
+        if let Some(ref ext) = sb_ext {
+            buf.extend_from_slice(ext);
+        }
 
         // Root group OH
         let mut root_links: Vec<LinkMessage> = Vec::new();
@@ -1524,7 +1635,7 @@ impl FileWriter {
             root_dl,
             &root_attrs,
             root_dense_blob.as_ref(),
-        ));
+        )?);
         if let Some(ref b) = root_link_blob {
             buf.extend_from_slice(&b.blob);
         }
@@ -1549,7 +1660,7 @@ impl FileWriter {
                 dl,
                 &g.attrs,
                 group_dense_blobs[gi].as_ref(),
-            ));
+            )?);
             if let Some(ref b) = link_blob {
                 buf.extend_from_slice(&b.blob);
             }
@@ -1571,7 +1682,8 @@ impl FileWriter {
             buf.extend_from_slice(&blob.data);
         }
 
-        debug_assert_eq!(buf.len(), cursor2);
+        debug_assert_eq!(buf.len(), data_end);
+        buf.resize(cursor2, 0);
         Ok(buf)
     }
 }
@@ -2145,7 +2257,8 @@ mod tests {
     }
 
     #[test]
-    fn file_writer_v4_superblock() {
+    fn file_writer_paged_file_uses_v3_superblock_and_fsinfo_extension() {
+        // This used to write superblock "version 4", which does not exist.
         let mut fw = FileWriter::new();
         fw.with_page_size(4096);
         fw.create_dataset("data").with_f64_data(&[1.0, 2.0]);
@@ -2153,8 +2266,31 @@ mod tests {
 
         let sig = signature::find_signature(&bytes).unwrap();
         let sb = Superblock::parse(&bytes, sig).unwrap();
-        assert_eq!(sb.version, 4, "expected superblock v4");
-        assert_eq!(sb.page_size, Some(4096));
+        assert_eq!(sb.version, 3);
+        assert_eq!(sb.superblock_extension_address, Some(48));
+        assert_eq!(bytes.len() % 4096, 0);
+        assert_eq!(sb.eof_address, bytes.len() as u64);
+        let ext = ObjectHeader::parse(&bytes, 48, 8, 8).unwrap();
+        let fsinfo = &ext.messages[0];
+        assert_eq!(fsinfo.msg_type, MessageType::Unknown(0x0017));
+        // Byte-for-byte what HDF5 2.0 writes for fs_strategy="page",
+        // fs_page_size=4096.
+        let mut expected = vec![1u8, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0];
+        expected.extend_from_slice(&4096u64.to_le_bytes());
+        expected.extend_from_slice(&[0, 0]);
+        expected.extend_from_slice(&[0xff; 8]);
+        assert_eq!(fsinfo.data, expected);
+        assert_eq!(fsinfo.flags, 0x14);
+        assert_eq!(read_dataset_f64(&bytes, "data"), vec![1.0, 2.0]);
+    }
+
+    #[test]
+    fn file_writer_rejects_page_sizes_libhdf5_would() {
+        for ps in [0u32, 511, MAX_FILE_SPACE_PAGE_SIZE + 1] {
+            let mut fw = FileWriter::new();
+            fw.with_page_size(ps);
+            assert!(fw.finish().is_err(), "page size {ps}");
+        }
     }
 
     #[test]

crates/clawhdf5-format/src/fill_value.rs

@@ -98,15 +98,50 @@ pub fn parse_fill_value(msg: &HeaderMessage) -> Result<Option<Vec<u8>>, FormatEr
 
 /// The fill value that applies to a dataset given its header messages. The new
 /// message wins over the old one when both are present.
+///
+/// A *shared* fill value message holds only a reference to the real message,
+/// which cannot be followed without the file: this returns
+/// [`FormatError::UnresolvedSharedMessage`] for one (it used to answer "zeros").
+/// Use [`dataset_fill_value_in`] when the file bytes are at hand.
 pub fn dataset_fill_value(messages: &[HeaderMessage]) -> Result<Option<Vec<u8>>, FormatError> {
+    fill_value_from(messages, |_| Err(FormatError::UnresolvedSharedMessage))
+}
+
+/// [`dataset_fill_value`] for a dataset in `file_data`, following a shared
+/// fill value message to where it lives: another object header, or the
+/// file's shared-message (SOHM) heap, as libhdf5 writes it when the file has
+/// a SOHM index for fill values.
+pub fn dataset_fill_value_in(
+    file_data: &[u8],
+    messages: &[HeaderMessage],
+    offset_size: u8,
+    length_size: u8,
+) -> Result<Option<Vec<u8>>, FormatError> {
+    fill_value_from(messages, |msg| {
+        crate::shared_message::message_data_with_sohm(file_data, msg, offset_size, length_size)
+            .map(|data| data.into_owned())
+    })
+}
+
+fn fill_value_from(
+    messages: &[HeaderMessage],
+    resolve_shared: impl Fn(&HeaderMessage) -> Result<Vec<u8>, FormatError>,
+) -> Result<Option<Vec<u8>>, FormatError> {
     for wanted in [MessageType::FillValue, MessageType::FillValueOld] {
         if let Some(msg) = messages.iter().find(|m| m.msg_type == wanted) {
-            if crate::shared_message::is_shared(msg.flags) {
-                // A shared fill value is legal but vanishingly rare; treat it
-                // as the default rather than misparsing the reference.
-                return Ok(None);
-            }
-            if let Some(value) = parse_fill_value(msg)? {
+            let value = if crate::shared_message::is_shared(msg.flags) {
+                let data = resolve_shared(msg)?;
+                parse_fill_value(&HeaderMessage {
+                    msg_type: msg.msg_type,
+                    size: data.len(),
+                    flags: msg.flags & !0x02,
+                    creation_order: msg.creation_order,
+                    data,
+                })?
+            } else {
+                parse_fill_value(msg)?
+            };
+            if let Some(value) = value {
                 return Ok(Some(value));
             }
         }
@@ -174,7 +209,7 @@ pub fn read_full_with_fill<E: From<FormatError>>(
     {
         return Err(FormatError::ExternalDataFilesUnsupported.into());
     }
-    let fill = dataset_fill_value(messages)?;
+    let fill = dataset_fill_value_in(file_data, messages, offset_size, length_size)?;
     if !has_storage(layout) {
         return Ok(filled_dataset(dataspace, elem_size, fill.as_deref())?);
     }

crates/clawhdf5-format/src/filter_pipeline.rs

@@ -19,8 +19,23 @@ pub const FILTER_SCALEOFFSET: u16 = 6;
 pub const FILTER_LZ4: u16 = 32004;
 /// Zstandard compression.
 pub const FILTER_ZSTD: u16 = 32015;
-/// Pcodec lossless numerical codec (clawhdf5 internal; not yet HDF5-registered).
-pub const FILTER_PCODEC: u16 = 32023;
+/// Pcodec lossless numerical codec — a **private, unregistered** clawhdf5
+/// filter. Pcodec has no ID in the HDF Group's filter registry (checked
+/// 2026-09-25, `hdf5_plugins/docs/RegisteredFilterPlugins.md`), so it uses an
+/// ID from the registry's testing/private range (256–511). No libhdf5 plugin
+/// decodes it: h5py/libhdf5 report the filter as unavailable. Only clawhdf5
+/// (with the `pcodec` feature) reads these datasets.
+pub const FILTER_PCODEC: u16 = 480;
+/// Filter name written with [`FILTER_PCODEC`].
+pub const FILTER_PCODEC_NAME: &str = "pcodec (clawhdf5 private)";
+/// The ID clawhdf5 up to 2.7.0 wrote pcodec under. It is registered to
+/// Granular BitRound (GBR), whose decode is a pass-through, so libhdf5 with
+/// that plugin would have returned the compressed bytes as data. Read as
+/// pcodec only when the filter is named exactly [`FILTER_PCODEC_LEGACY_NAME`],
+/// the name those versions wrote; never written.
+pub const FILTER_PCODEC_LEGACY: u16 = 32023;
+/// The filter name clawhdf5 up to 2.7.0 wrote with [`FILTER_PCODEC_LEGACY`].
+pub const FILTER_PCODEC_LEGACY_NAME: &str = "pcodec";
 
 /// Description of a single filter in a pipeline.
 #[derive(Debug, Clone, PartialEq)]

crates/clawhdf5-format/src/filters_szip.rs

@@ -1,19 +1,39 @@
 //! SZIP (libaec Adaptive Entropy Coding) decompression.
 //!
 //! Gated by the `szip` feature which links against the system libaec library.
+//!
+//! libhdf5's SZIP filter (`H5Zszip.c`) prefixes each chunk with its
+//! uncompressed size and hands the rest to szlib's `SZ_BufftoBuffDecompress`.
+//! libaec implements that call (`sz_compat.c`) on top of `aec_buffer_decode`
+//! with some reshaping — 32/64-bit samples are coded as byte planes of 8-bit
+//! samples, and scanlines that are not a whole number of blocks are padded —
+//! which [`szip_decompress`] reproduces so its output matches libhdf5's.
 
 #[cfg(not(feature = "std"))]
 use alloc::vec::Vec;
 
 use crate::error::FormatError;
 
-/// Decompress SZIP-compressed data using libaec.
+/// `SZ_MSB_OPTION_MASK`: samples are big-endian.
+#[cfg(feature = "szip")]
+const SZ_MSB_OPTION_MASK: u32 = 16;
+/// `SZ_NN_OPTION_MASK`: nearest-neighbour preprocessing.
+#[cfg(feature = "szip")]
+const SZ_NN_OPTION_MASK: u32 = 32;
+
+/// Decompress one SZIP-filtered chunk.
 ///
-/// `cd` is the HDF5 SZIP filter client data (matches `H5Z_SZIP_PARM_*` indices):
-///   cd[0] = options mask  (`H5_SZIP_NN_OPTION_MASK = 0x20` enables NN preprocessing)
-///   cd[1] = pixels per block (H5Z_SZIP_PARM_PPB; 8, 10, 16, or 32)
-///   cd[2] = bits per sample  (H5Z_SZIP_PARM_BPP; element bit width)
-///   cd[3] = pixels per scan line (H5Z_SZIP_PARM_PPS; informational only)
+/// `cd` is the HDF5 SZIP filter client data (`H5Z_SZIP_PARM_*` indices):
+///   cd[0] = options mask (`SZ_*_OPTION_MASK`: 16 = MSB byte order,
+///           32 = nearest-neighbour preprocessing; K13/EC/LSB/RAW bits carry
+///           no decoding information for libaec)
+///   cd[1] = pixels per block
+///   cd[2] = bits per pixel (sample precision, rounded up to 32 or 64 above
+///           24 by libhdf5)
+///   cd[3] = pixels per scanline
+///
+/// The chunk is a 4-byte little-endian uncompressed size followed by the
+/// szlib stream.
 pub(crate) fn szip_decompress(
     _data: &[u8],
     _cd: &[u32],
@@ -33,62 +53,174 @@ pub(crate) fn szip_decompress(
 
 #[cfg(feature = "szip")]
 fn szip_decode_impl(data: &[u8], cd: &[u32], chunk_size: usize) -> Result<Vec<u8>, FormatError> {
-    if cd.len() < 3 {
-        return Err(FormatError::ChunkedReadError(
-            "szip: missing client data".into(),
-        ));
+    let err = |m: &str| FormatError::ChunkedReadError(format!("szip: {m}"));
+    if cd.len() < 4 {
+        return Err(err("missing client data"));
     }
     let options = cd[0];
-    let pixels_per_block = cd[1];
-    let bits_per_sample = cd[2]; // H5Z_SZIP_PARM_BPP
-    if bits_per_sample == 0 || bits_per_sample > 32 {
-        return Err(FormatError::ChunkedReadError(
-            "szip: invalid bits per sample".into(),
-        ));
+    let pixels_per_block = cd[1] as usize;
+    let bits_per_pixel = cd[2];
+    let pixels_per_scanline = cd[3] as usize;
+    if !(1..=32).contains(&bits_per_pixel) && bits_per_pixel != 64 {
+        return Err(err("invalid bits per sample"));
     }
-    if chunk_size == 0 {
-        return Err(FormatError::ChunkedReadError(
-            "szip: unknown output size".into(),
-        ));
+    if pixels_per_block == 0 || pixels_per_scanline == 0 {
+        return Err(err("invalid block or scanline size"));
     }
-    if data.is_empty() {
-        return Err(FormatError::ChunkedReadError("szip: empty input".into()));
+    if data.len() < 4 {
+        return Err(err("chunk too short"));
     }
+    // H5Zszip.c: UINT32DECODE of the uncompressed size, then the stream.
+    let dest_len = u32::from_le_bytes([data[0], data[1], data[2], data[3]]) as usize;
+    let limit = if chunk_size != 0 {
+        chunk_size
+    } else {
+        crate::filters::MAX_DECOMPRESS_SIZE
+    };
+    if dest_len > limit {
+        return Err(err("declared size exceeds chunk size"));
+    }
+    let stream = &data[4..];
 
-    // Map HDF5 option mask to libaec flags.
-    // HDF5 always stores SZIP data in MSB order, so AEC_DATA_MSB is unconditional.
-    // H5_SZIP_NN_OPTION_MASK (0x20): NN differential preprocessing.
-    let mut flags: u32 = libaec_sys::AEC_DATA_MSB;
-    if options & 0x20 != 0 {
+    // --- libaec sz_compat.c: SZ_BufftoBuffDecompress ---
+    let rsi = pixels_per_scanline.div_ceil(pixels_per_block);
+    let mut flags = 0;
+    if options & SZ_MSB_OPTION_MASK != 0 {
+        flags |= libaec_sys::AEC_DATA_MSB;
+    }
+    if options & SZ_NN_OPTION_MASK != 0 {
         flags |= libaec_sys::AEC_DATA_PREPROCESS;
     }
+    let pad_scanline = !pixels_per_scanline.is_multiple_of(pixels_per_block);
+    let deinterleave = bits_per_pixel == 32 || bits_per_pixel == 64;
+    let bits_per_sample = if deinterleave { 8 } else { bits_per_pixel };
+    let pixel_size = match bits_per_sample {
+        17.. => 4,
+        9.. => 2,
+        _ => 1,
+    };
+    let scanlines = (dest_len / pixel_size).div_ceil(pixels_per_scanline);
+    let buf_size = if pad_scanline {
+        rsi.checked_mul(pixels_per_block)
+            .and_then(|n| n.checked_mul(pixel_size))
+            .and_then(|n| n.checked_mul(scanlines))
+            .filter(|&n| n <= crate::filters::MAX_DECOMPRESS_SIZE.max(limit))
+            .ok_or_else(|| err("scanline padding too large"))?
+    } else {
+        dest_len
+    };
 
-    let mut out = vec![0u8; chunk_size];
+    let mut buf = vec![0u8; buf_size];
     let mut strm = libaec_sys::AecStream::zeroed();
-    strm.next_in = data.as_ptr();
-    strm.avail_in = data.len();
-    strm.next_out = out.as_mut_ptr();
-    strm.avail_out = chunk_size;
+    strm.next_in = stream.as_ptr();
+    strm.avail_in = stream.len();
+    strm.next_out = buf.as_mut_ptr();
+    strm.avail_out = buf_size;
     strm.bits_per_sample = bits_per_sample;
-    strm.block_size = pixels_per_block;
-    strm.rsi = 128; // HDF5 default: 128 blocks per reference sample interval
+    strm.block_size = pixels_per_block as u32;
+    strm.rsi = rsi as u32;
     strm.flags = flags;
-
+    // SAFETY: next_in/avail_in and next_out/avail_out describe live buffers
+    // (`stream` and `buf`) that outlive the call.
     let result = unsafe { libaec_sys::aec_buffer_decode(&mut strm) };
     if result != 0 {
         return Err(FormatError::DecompressionError(format!(
             "szip: libaec error {result}"
         )));
     }
-    let decoded_len = chunk_size - strm.avail_out;
-    out.truncate(decoded_len);
+    let mut total_out = strm.total_out;
+    if pad_scanline {
+        let line = pixels_per_scanline * pixel_size;
+        let padded_line = rsi * pixels_per_block * pixel_size;
+        // remove_padding: compact each padded line down to `line` bytes.
+        let mut i = line;
+        let mut j = padded_line;
+        while j < total_out {
+            let end = (j + line).min(buf.len());
+            buf.copy_within(j..end, i);
+            i += line;
+            j += padded_line;
+        }
+        total_out = scanlines * line;
+    }
+    if total_out < dest_len {
+        return Err(err("stream decoded to fewer bytes than declared"));
+    }
+    buf.truncate(dest_len);
+    if deinterleave {
+        // deinterleave_buffer: byte planes back into words.
+        let w = (bits_per_pixel / 8) as usize;
+        let n = dest_len / w;
+        let mut out = vec![0u8; dest_len];
+        for i in 0..n {
+            for j in 0..w {
+                out[i * w + j] = buf[j * n + i];
+            }
+        }
         Ok(out)
+    } else {
+        Ok(buf)
+    }
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
 
+    #[cfg(feature = "szip")]
+    fn unhex(s: &str) -> Vec<u8> {
+        (0..s.len())
+            .step_by(2)
+            .map(|i| u8::from_str_radix(&s[i..i + 2], 16).unwrap())
+            .collect()
+    }
+
+    /// SZIP chunks written by libhdf5, decoded exactly as libhdf5 decodes
+    /// them. Each case: fixture, chunk byte offset and size (from h5py's
+    /// `get_chunk_info`), the filter's cd_values, and the chunk's values as
+    /// h5py reads them (file byte order, hex). Before the fix every one of
+    /// these came back as garbage or zeros (or "invalid bits per sample" for
+    /// 64-bit): the 4-byte size prefix was fed to libaec, 32/64-bit samples
+    /// were not de-interleaved from byte planes, the reference sample
+    /// interval was fixed at 128 instead of derived from the scanline, padded
+    /// scanlines were not unpadded, and LE data was decoded as MSB.
+    #[cfg(feature = "szip")]
+    #[test]
+    fn szip_decodes_libhdf5_chunks_exactly() {
+        /// (name, file, chunk offset, chunk size, cd_values, decoded hex)
+        type Case<'a> = (&'a str, &'a [u8], usize, usize, [u32; 4], &'a str);
+        let noencoder: &[u8] = include_bytes!("../tests/fixtures/filters/noencoder.h5");
+        let le_data: &[u8] = include_bytes!("../tests/fixtures/filters/le_data.h5");
+        let h5py: &[u8] = include_bytes!("../tests/fixtures/filters/szip_h5py.h5");
+        #[rustfmt::skip]
+        let cases: &[Case] = &[
+            // <i4, 10 px/scanline over 4 px/block: padded scanlines + byte planes.
+            ("noencoder /noencoder_szip_dset.h5", noencoder, 6040, 16, [168, 4, 32, 10],
+             "00000000010000000200000003000000040000000500000006000000070000000800000009000000"),
+            // <f4, LSB + NN.
+            ("le_data /Szip_float_data_le", le_data, 55224, 48, [169, 4, 32, 12],
+             "abaaaa3eabaa2a3f0000803fabaa2a3f0000803fabaaaa3f0000803fabaaaa3f5555d53fabaaaa3f5555d53f00000040"),
+            // >f4, MSB + NN.
+            ("le_data /Szip_float_data_be", le_data, 55396, 48, [177, 4, 32, 12],
+             "3eaaaaab3f2aaaab3f8000003f2aaaab3f8000003faaaaab3f8000003faaaaab3fd555553faaaaab3fd5555540000000"),
+            // <f8 (64-bit), NN.
+            ("szip_h5py /f8", h5py, 4016, 100, [169, 8, 64, 10],
+             "00000000000008c000000000000008c000000000000008c000000000000008c000000000000004c000000000000004c000000000000004c000000000000004c000000000000000c000000000000000c000000000000000c000000000000000c0000000000000f8bf000000000000f8bf000000000000f8bf000000000000f8bf000000000000f0bf000000000000f0bf000000000000f0bf000000000000f0bf000000000000e0bf000000000000e0bf000000000000e0bf000000000000e0bf0000000000000000000000000000000000000000000000000000000000000000000000000000e03f000000000000e03f000000000000e03f000000000000e03f000000000000f03f000000000000f03f000000000000f03f000000000000f03f000000000000f83f000000000000f83f000000000000f83f000000000000f83f"),
+            // <i8 (64-bit), entropy coding without NN.
+            ("szip_h5py /i8", h5py, 4188, 53, [141, 4, 64, 10],
+             "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000300000000000000030000000000000003000000000000000300000000000000030000000000000003000000000000000300000000000000030000000000000006000000000000000600000000000000060000000000000006000000000000000600000000000000060000000000000006000000000000000600000000000000090000000000000009000000000000000900000000000000090000000000000009000000000000000900000000000000090000000000000009000000000000000c000000000000000c000000000000000c000000000000000c000000000000000c000000000000000c000000000000000c000000000000000c00000000000000"),
+            // <u2, 35 px/scanline over 8 px/block: padded scanlines, 16-bit samples.
+            ("szip_h5py /u2", h5py, 4308, 43, [169, 8, 16, 35],
+             "00000000000000006100610061006100c200c200c200c20023012301230123018401840184018401e501e501e501e5014602460246024602a702a702a702a702080308030803"),
+        ];
+        for (name, file, off, len, cd, want) in cases {
+            let want = unhex(want);
+            let got = szip_decompress(&file[*off..off + len], cd, want.len())
+                .unwrap_or_else(|e| panic!("{name}: {e:?}"));
+            assert_eq!(got, want, "{name}");
+        }
+    }
+
     #[test]
     fn szip_disabled_returns_unsupported() {
         #[cfg(not(feature = "szip"))]
@@ -132,6 +264,8 @@ mod tests {
         assert_eq!(rc, 0, "aec_buffer_encode failed: {rc}");
         let enc_len = encoded.len() - enc.avail_out;
         encoded.truncate(enc_len);
+        // H5Zszip.c prefixes the stream with the uncompressed size.
+        encoded.splice(0..0, (original.len() as u32).to_le_bytes());
 
         // Decode through our public interface.
         // cd[0]=0 (no NN bit 0x20), cd[1]=8 (ppb), cd[2]=8 (bpp), cd[3]=1024 (pps).
@@ -163,6 +297,8 @@ mod tests {
         assert_eq!(rc, 0, "aec_buffer_encode with NN failed: {rc}");
         let enc_len = encoded.len() - enc.avail_out;
         encoded.truncate(enc_len);
+        // H5Zszip.c prefixes the stream with the uncompressed size.
+        encoded.splice(0..0, (original.len() as u32).to_le_bytes());
 
         // cd[0] = 0x20 (H5_SZIP_NN_OPTION_MASK) → decoder must set AEC_DATA_PREPROCESS.
         let cd = [0x20u32, 8, 8, 1024];

crates/clawhdf5-format/src/fixed_array.rs

@@ -6,6 +6,7 @@ extern crate alloc;
 #[cfg(not(feature = "std"))]
 use alloc::{format, vec, vec::Vec};
 
+use crate::chunk_grid::ChunkGrid;
 use crate::chunked_read::ChunkInfo;
 use crate::error::FormatError;
 
@@ -151,13 +152,13 @@ pub fn read_fixed_array_chunks(
     file_data: &[u8],
     header: &FixedArrayHeader,
     dataset_dims: &[u64],
+    max_dims: Option<&[u64]>,
     chunk_dimensions: &[u32],
     element_size: u32,
     offset_size: u8,
     _length_size: u8,
 ) -> Result<Vec<ChunkInfo>, FormatError> {
     let db_offset = header.data_block_address as usize;
-    let rank = chunk_dimensions.len();
 
     // Parse data block header: FADB(4) + version(1) + client_id(1) + header_address(offset_size)
     let db_header_size = 4 + 1 + 1 + offset_size as usize;
@@ -198,19 +199,10 @@ pub fn read_fixed_array_chunks(
             ))
     };
 
-    // Compute chunk offsets based on index.
-    // Chunks are stored in row-major order within the dataset space.
-    let mut num_chunks_per_dim = Vec::with_capacity(rank);
-    for d_idx in 0..rank {
-        let ch_dim = chunk_dimensions[d_idx] as u64;
-        if ch_dim == 0 {
-            return Err(FormatError::ChunkedReadError(
-                "chunk dimension is zero".into(),
-            ));
-        }
-        let ds_dim = dataset_dims[d_idx];
-        num_chunks_per_dim.push(ds_dim.div_ceil(ch_dim));
-    }
+    // The index is laid out over the chunk grid of the *maximum* dimensions
+    // (row-major), so a dataset smaller than its maxshape has gaps.
+    let dims_u64: Vec<u64> = chunk_dimensions.iter().map(|&d| d as u64).collect();
+    let grid = ChunkGrid::fixed_array(dataset_dims, max_dims, &dims_u64)?;
 
     let chunk_byte_size: u64 =
         chunk_dimensions.iter().map(|&d| d as u64).product::<u64>() * element_size as u64;
@@ -226,7 +218,11 @@ pub fn read_fixed_array_chunks(
                 header.element_size,
                 chunk_byte_size,
             )? {
-                let offsets = index_to_chunk_offsets(i, &num_chunks_per_dim, chunk_dimensions);
+                // A slot beyond the current extent is ignored, as the
+                // library does.
+                let Some(offsets) = grid.offsets(i as u64) else {
+                    return Ok(());
+                };
                 chunks.push(ChunkInfo {
                     chunk_size,
                     filter_mask,
@@ -367,27 +363,6 @@ fn parse_fa_element(
     }
 }
 
-/// Convert a linear chunk index to N-dimensional chunk offsets in dataset space.
-fn index_to_chunk_offsets(
-    index: usize,
-    num_chunks_per_dim: &[u64],
-    chunk_dimensions: &[u32],
-) -> Vec<u64> {
-    let rank = num_chunks_per_dim.len();
-    let mut offsets = vec![0u64; rank];
-    let mut remaining = index as u64;
-    for d in (0..rank).rev() {
-        let nchunks = num_chunks_per_dim[d];
-        if nchunks == 0 {
-            continue;
-        }
-        let chunk_idx = remaining % nchunks;
-        remaining /= nchunks;
-        offsets[d] = chunk_idx * chunk_dimensions[d] as u64;
-    }
-    offsets
-}
-
 /// Read a variable-length little-endian unsigned integer.
 fn read_variable_length(data: &[u8], size: usize) -> Result<u64, FormatError> {
     if size > 8 || data.len() < size {
@@ -416,44 +391,21 @@ mod tests {
 
     #[test]
     fn index_to_offsets_1d() {
-        let num_chunks = vec![5u64];
-        let chunk_dims = vec![20u32];
-        assert_eq!(index_to_chunk_offsets(0, &num_chunks, &chunk_dims), vec![0]);
-        assert_eq!(
-            index_to_chunk_offsets(1, &num_chunks, &chunk_dims),
-            vec![20]
-        );
-        assert_eq!(
-            index_to_chunk_offsets(4, &num_chunks, &chunk_dims),
-            vec![80]
-        );
+        let g = ChunkGrid::fixed_array(&[100], None, &[20]).unwrap();
+        assert_eq!(g.offsets(0).unwrap(), vec![0]);
+        assert_eq!(g.offsets(1).unwrap(), vec![20]);
+        assert_eq!(g.offsets(4).unwrap(), vec![80]);
     }
 
     #[test]
     fn index_to_offsets_2d() {
         // 10x6 dataset with 4x3 chunks => ceil(10/4)=3, ceil(6/3)=2 => 6 chunks
-        let num_chunks = vec![3u64, 2];
-        let chunk_dims = vec![4u32, 3];
-        assert_eq!(
-            index_to_chunk_offsets(0, &num_chunks, &chunk_dims),
-            vec![0, 0]
-        );
-        assert_eq!(
-            index_to_chunk_offsets(1, &num_chunks, &chunk_dims),
-            vec![0, 3]
-        );
-        assert_eq!(
-            index_to_chunk_offsets(2, &num_chunks, &chunk_dims),
-            vec![4, 0]
-        );
-        assert_eq!(
-            index_to_chunk_offsets(3, &num_chunks, &chunk_dims),
-            vec![4, 3]
-        );
-        assert_eq!(
-            index_to_chunk_offsets(5, &num_chunks, &chunk_dims),
-            vec![8, 3]
-        );
+        let g = ChunkGrid::fixed_array(&[10, 6], None, &[4, 3]).unwrap();
+        assert_eq!(g.offsets(0).unwrap(), vec![0, 0]);
+        assert_eq!(g.offsets(1).unwrap(), vec![0, 3]);
+        assert_eq!(g.offsets(2).unwrap(), vec![4, 0]);
+        assert_eq!(g.offsets(3).unwrap(), vec![4, 3]);
+        assert_eq!(g.offsets(5).unwrap(), vec![8, 3]);
     }
 
     #[test]
@@ -517,7 +469,7 @@ mod tests {
 
         let read = |f: &[u8], fahd: usize| -> Result<Vec<ChunkInfo>, FormatError> {
             let h = FixedArrayHeader::parse(f, fahd, 8, 8)?;
-            read_fixed_array_chunks(f, &h, &[60], &[20], 8, 8, 8)
+            read_fixed_array_chunks(f, &h, &[60], None, &[20], 8, 8, 8)
         };
 
         let (clean, fahd) = build();
@@ -562,7 +514,7 @@ mod tests {
         let db = 0x100usize;
         buf[db..db + 4].copy_from_slice(b"FADB");
         let header = FixedArrayHeader::parse(&buf, fahd, 8, 8).unwrap();
-        let r = read_fixed_array_chunks(&buf, &header, &[100], &[20], 8, 8, 8);
+        let r = read_fixed_array_chunks(&buf, &header, &[100], None, &[20], 8, 8, 8);
         assert!(r.is_err());
     }
 
@@ -579,7 +531,7 @@ mod tests {
         stamp_checksum(&mut buf, fahd, fahd + 24);
         buf[0x80..0x84].copy_from_slice(b"FADB");
         let header = FixedArrayHeader::parse(&buf, fahd, 8, 8).unwrap();
-        let r = read_fixed_array_chunks(&buf, &header, &[100], &[20], 8, 8, 8);
+        let r = read_fixed_array_chunks(&buf, &header, &[100], None, &[20], 8, 8, 8);
         assert!(r.is_err());
     }
 
@@ -602,7 +554,7 @@ mod tests {
             data_block_address: (usize::MAX - 4) as u64,
         };
         let buf = vec![0u8; 64];
-        let r = read_fixed_array_chunks(&buf, &header, &[100], &[20], 8, 8, 8);
+        let r = read_fixed_array_chunks(&buf, &header, &[100], None, &[20], 8, 8, 8);
         assert!(r.is_err());
     }
 
@@ -664,6 +616,7 @@ mod tests {
             &file_data,
             &header,
             &ds_dims,
+            None,
             &chunk_dims,
             8,
             offset_size,
@@ -740,6 +693,7 @@ mod tests {
             &file_data,
             &header,
             &ds_dims,
+            None,
             &chunk_dims,
             8,
             offset_size,
@@ -840,6 +794,7 @@ mod tests {
             &file_data,
             &header,
             &ds_dims,
+            None,
             &chunk_dims,
             8,
             offset_size,

crates/clawhdf5-format/src/float16.rs

@@ -0,0 +1,155 @@
+//! IEEE-754 half precision (binary16) conversions.
+//!
+//! Pure integer bit manipulation, so it works under `no_std` and needs no
+//! `libm`. The writer ([`crate::type_builders::DatasetBuilder::with_f16_data`]),
+//! the reader and `clawhdf5-agent`'s half-precision embedding store all use
+//! these two functions, so a value rounded in memory is bit-for-bit the value
+//! that reads back from the file.
+
+/// Largest finite half-precision value. Anything larger in magnitude rounds
+/// to infinity.
+pub const F16_MAX: f32 = 65504.0;
+
+/// Convert an `f32` to the bit pattern of the nearest half-precision value,
+/// rounding ties to even (the IEEE default, and what numpy and the `half`
+/// crate do).
+///
+/// Values beyond ±[`F16_MAX`] become ±infinity, values too small for a
+/// subnormal become signed zero, and NaN stays NaN (quiet, payload
+/// truncated).
+pub fn f32_to_f16_bits(value: f32) -> u16 {
+    let x = value.to_bits();
+    let sign = (x >> 16) & 0x8000;
+    let exp = x & 0x7F80_0000;
+    let man = x & 0x007F_FFFF;
+
+    // Infinity and NaN.
+    if exp == 0x7F80_0000 {
+        let quiet_nan = if man == 0 { 0 } else { 0x0200 };
+        return (sign | 0x7C00 | quiet_nan | (man >> 13)) as u16;
+    }
+
+    let half_exp = ((exp >> 23) as i32) - 127 + 15;
+
+    // Too large: infinity.
+    if half_exp >= 0x1F {
+        return (sign | 0x7C00) as u16;
+    }
+
+    // Subnormal half, or zero.
+    if half_exp <= 0 {
+        if 14 - half_exp > 24 {
+            return sign as u16;
+        }
+        let man = man | 0x0080_0000; // implicit leading bit
+        let shift = (14 - half_exp) as u32;
+        let mut half_man = man >> shift;
+        let round_bit = 1u32 << (shift - 1);
+        // Round half to even: up if above half, or exactly half and odd.
+        if (man & round_bit) != 0 && (man & (3 * round_bit - 1)) != 0 {
+            half_man += 1;
+        }
+        return (sign | half_man) as u16;
+    }
+
+    // Normal half. A mantissa carry correctly rolls into the exponent (and
+    // from the largest finite value into infinity).
+    let half = sign | ((half_exp as u32) << 10) | (man >> 13);
+    let round_bit = 0x0000_1000;
+    if (man & round_bit) != 0 && (man & (3 * round_bit - 1)) != 0 {
+        (half + 1) as u16
+    } else {
+        half as u16
+    }
+}
+
+/// Convert the bit pattern of a half-precision value to `f32` (exact: every
+/// half value is representable as an `f32`).
+pub fn f16_bits_to_f32(h: u16) -> f32 {
+    let h = h as u32;
+    let sign = (h & 0x8000) << 16;
+    let exp = (h >> 10) & 0x1f;
+    let mant = h & 0x3ff;
+    let bits = if exp == 0 {
+        if mant == 0 {
+            sign // signed zero
+        } else {
+            // Subnormal: normalize into an f32 normal.
+            let mut e: i32 = -1;
+            let mut m = mant;
+            loop {
+                e += 1;
+                m <<= 1;
+                if m & 0x400 != 0 {
+                    break;
+                }
+            }
+            let m = m & 0x3ff;
+            sign | (((127 - 15 - e) as u32) << 23) | (m << 13)
+        }
+    } else if exp == 0x1f {
+        sign | 0x7f80_0000 | (mant << 13) // inf / NaN
+    } else {
+        sign | ((exp + 127 - 15) << 23) | (mant << 13)
+    };
+    f32::from_bits(bits)
+}
+
+/// Round an `f32` to the nearest half-precision value, returned as `f32`.
+pub fn round_to_f16(value: f32) -> f32 {
+    f16_bits_to_f32(f32_to_f16_bits(value))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn every_half_value_round_trips() {
+        for bits in 0..=u16::MAX {
+            let v = f16_bits_to_f32(bits);
+            if v.is_nan() {
+                assert!(f16_bits_to_f32(f32_to_f16_bits(v)).is_nan(), "{bits:#06x}");
+            } else {
+                assert_eq!(f32_to_f16_bits(v), bits, "{bits:#06x} -> {v}");
+            }
+        }
+    }
+
+    #[test]
+    fn matches_the_half_crate() {
+        // Every 257th f32 bit pattern (~16.7M values) covers every exponent,
+        // the subnormal range, both signs, ties and the overflow boundary.
+        let mut bits: u32 = 0;
+        loop {
+            let v = f32::from_bits(bits);
+            let ours = f32_to_f16_bits(v);
+            let theirs = half::f16::from_f32(v);
+            if v.is_nan() {
+                assert!(theirs.is_nan() && f16_bits_to_f32(ours).is_nan());
+            } else {
+                assert_eq!(ours, theirs.to_bits(), "{bits:#010x} ({v:e})");
+                assert_eq!(f16_bits_to_f32(ours).to_bits(), theirs.to_f32().to_bits());
+            }
+            match bits.checked_add(257) {
+                Some(b) => bits = b,
+                None => break,
+            }
+        }
+    }
+
+    #[test]
+    fn rounds_ties_to_even_and_saturates_to_infinity() {
+        // 1 + 2^-11 is exactly halfway between 1.0 and the next half (1 + 2^-10).
+        assert_eq!(round_to_f16(1.0 + 2f32.powi(-11)), 1.0);
+        assert_eq!(
+            round_to_f16(1.0 + 3.0 * 2f32.powi(-11)),
+            1.0 + 2.0 * 2f32.powi(-10)
+        );
+        assert_eq!(round_to_f16(F16_MAX), F16_MAX);
+        assert_eq!(round_to_f16(65520.0), f32::INFINITY); // halfway to 2^16 rounds up
+        assert_eq!(round_to_f16(-1e9), f32::NEG_INFINITY);
+        assert_eq!(round_to_f16(1e-9).to_bits(), 0);
+        assert_eq!(round_to_f16(-1e-9).to_bits(), (-0.0f32).to_bits());
+    }
+}

crates/clawhdf5-format/src/lib.rs

@@ -54,6 +54,7 @@ pub mod btree_v1;
 pub mod btree_v2;
 pub mod checksum;
 pub mod chunk_cache;
+mod chunk_grid;
 pub mod chunk_index;
 pub mod chunked_read;
 pub mod chunked_write;
@@ -72,6 +73,7 @@ pub mod filter_pipeline;
 pub mod filters;
 mod filters_szip;
 pub mod fixed_array;
+pub mod float16;
 pub mod fractal_heap;
 pub mod global_heap;
 pub mod group_info;

crates/clawhdf5-format/src/object_header.rs

@@ -146,12 +146,7 @@ impl ObjectHeader {
             ensure_len(data, pos, msg_data_size)?;
             let msg_type = MessageType::from_u16(msg_type_raw);
 
-            // Check if unknown + must-understand (bit 3 of msg_flags)
-            if let MessageType::Unknown(id) = msg_type
-                && msg_flags & 0x08 != 0
-            {
-                return Err(FormatError::UnsupportedMessage(id));
-            }
+            check_unknown_message(msg_type, msg_flags)?;
 
             if msg_type != MessageType::Nil {
                 messages.push(HeaderMessage {
@@ -229,11 +224,7 @@ impl ObjectHeader {
 
             let msg_type = MessageType::from_u16(msg_type_raw);
 
-            if let MessageType::Unknown(id) = msg_type
-                && msg_flags & 0x08 != 0
-            {
-                return Err(FormatError::UnsupportedMessage(id));
-            }
+            check_unknown_message(msg_type, msg_flags)?;
 
             if msg_type != MessageType::Nil {
                 messages.push(HeaderMessage {
@@ -424,11 +415,7 @@ impl ObjectHeader {
 
             let msg_type = MessageType::from_u16(msg_type_raw);
 
-            if let MessageType::Unknown(id) = msg_type
-                && msg_flags & 0x08 != 0
-            {
-                return Err(FormatError::UnsupportedMessage(id));
-            }
+            check_unknown_message(msg_type, msg_flags)?;
 
             let msg_data = data[pos..pos + msg_data_size].to_vec();
 
@@ -509,6 +496,24 @@ impl ObjectHeader {
     }
 }
 
+/// Header message flag bit 7: fail if the message is unknown, always.
+const MSG_FLAG_FAIL_IF_UNKNOWN_ALWAYS: u8 = 0x80;
+
+/// Refuse an unknown message the file says no reader may skip.
+///
+/// The parser only ever reads, so bit 3 (fail only when opened for writing)
+/// is ignored, as libhdf5 ignores it for a read-only open; bit 7 fails
+/// regardless of access mode. This had the two the wrong way round, failing
+/// objects libhdf5 reads and reading ones it refuses (`tbogus.h5`).
+fn check_unknown_message(msg_type: MessageType, msg_flags: u8) -> Result<(), FormatError> {
+    match msg_type {
+        MessageType::Unknown(id) if msg_flags & MSG_FLAG_FAIL_IF_UNKNOWN_ALWAYS != 0 => {
+            Err(FormatError::UnsupportedMessage(id))
+        }
+        _ => Ok(()),
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -632,14 +637,38 @@ mod tests {
     }
 
     #[test]
-    fn parse_v1_unknown_must_understand_errors() {
-        // Bit 3 of msg_flags = must understand
-        let messages = [(0x00FFu16, &[0xAA][..], 0x08u8)];
+    fn parse_v1_unknown_fail_always_errors() {
+        // Bit 7 of msg_flags = fail if unknown, whatever the access mode.
+        let messages = [(0x00FFu16, &[0xAA][..], 0x80u8)];
         let data = build_v1_header(&messages, 8, 8);
         let err = ObjectHeader::parse(&data, 0, 8, 8).unwrap_err();
         assert_eq!(err, FormatError::UnsupportedMessage(0x00FF));
     }
 
+    #[test]
+    fn parse_v1_unknown_fail_on_write_is_ignored_when_reading() {
+        // Bit 3 = fail if unknown *and the file is opened for writing*. This
+        // parser only reads, so libhdf5 (read-only) opens such an object and
+        // so must we. Bits 4/5 (mark if unknown / was unknown) never fail.
+        for flags in [0x08u8, 0x10, 0x20, 0x38] {
+            let messages = [(0x00FFu16, &[0xAA][..], flags)];
+            let data = build_v1_header(&messages, 8, 8);
+            let hdr = ObjectHeader::parse(&data, 0, 8, 8).unwrap();
+            assert_eq!(hdr.messages[0].msg_type, MessageType::Unknown(0x00FF));
+        }
+    }
+
+    #[test]
+    fn parse_v2_unknown_message_flags() {
+        let data = build_v2_header(0x00, &[(0xF0, &[1, 2], 0x08)], None);
+        assert!(ObjectHeader::parse(&data, 0, 8, 8).is_ok());
+        let data = build_v2_header(0x00, &[(0xF0, &[1, 2], 0x80)], None);
+        assert_eq!(
+            ObjectHeader::parse(&data, 0, 8, 8).unwrap_err(),
+            FormatError::UnsupportedMessage(0xF0)
+        );
+    }
+
     #[test]
     fn parse_v2_no_timestamps_one_message() {
         let data = build_v2_header(0x00, &[(0x01, &[10, 20], 0)], None);

crates/clawhdf5-format/src/object_header_writer.rs

@@ -1,11 +1,17 @@
 //! Object header writer for v2 format.
 
 #[cfg(not(feature = "std"))]
-use alloc::vec::Vec;
+use alloc::{format, vec::Vec};
 
 use crate::checksum::jenkins_lookup3;
+use crate::error::FormatError;
 use crate::message_type::MessageType;
 
+/// Largest message payload a v2 object header can describe: the per-message
+/// size field is 2 bytes. A bigger message cannot be encoded at all — writing
+/// its size truncated to 16 bits produced files libhdf5 refuses.
+pub const MAX_MESSAGE_SIZE: usize = u16::MAX as usize;
+
 /// Writer for v2 object headers with proper checksums.
 pub struct ObjectHeaderWriter {
     messages: Vec<(MessageType, Vec<u8>, u8)>, // (type, data, msg_flags)
@@ -30,7 +36,22 @@ impl ObjectHeaderWriter {
     }
 
     /// Serialize the complete v2 object header (OHDR + messages + checksum).
-    pub fn serialize(&self) -> Vec<u8> {
+    ///
+    /// Fails with [`FormatError::SerializationError`] when a message is larger
+    /// than [`MAX_MESSAGE_SIZE`] (e.g. an attribute over ~64 KiB, which would
+    /// need dense attribute storage), rather than writing a corrupt header.
+    pub fn serialize(&self) -> Result<Vec<u8>, FormatError> {
+        if let Some((msg_type, data, _)) = self
+            .messages
+            .iter()
+            .find(|(_, data, _)| data.len() > MAX_MESSAGE_SIZE)
+        {
+            return Err(FormatError::SerializationError(format!(
+                "{msg_type:?} message is {} bytes; an object header message holds at most \
+                 {MAX_MESSAGE_SIZE} bytes",
+                data.len()
+            )));
+        }
         // Calculate total message bytes: each message has type(1) + size(2) + flags(1) + data
         let msg_bytes_total: usize = self
             .messages
@@ -80,7 +101,7 @@ impl ObjectHeaderWriter {
         let checksum = jenkins_lookup3(&buf);
         buf.extend_from_slice(&checksum.to_le_bytes());
 
-        buf
+        Ok(buf)
     }
 }
 
@@ -125,15 +146,22 @@ impl BatchObjectHeaderWriter {
 
     /// Compute the serialized size of each header without actually serializing.
     /// Returns sizes in the same order as headers were added.
-    pub fn compute_sizes(&self) -> Vec<usize> {
-        self.headers.iter().map(|h| h.serialize().len()).collect()
+    pub fn compute_sizes(&self) -> Result<Vec<usize>, FormatError> {
+        self.headers
+            .iter()
+            .map(|h| h.serialize().map(|b| b.len()))
+            .collect()
     }
 
     /// Serialize all headers into a single contiguous buffer.
     /// Returns `(combined_bytes, offsets)` where `offsets[i]` is the byte
     /// offset of header `i` within the combined buffer.
-    pub fn serialize_all(&self) -> (Vec<u8>, Vec<usize>) {
-        let serialized: Vec<Vec<u8>> = self.headers.iter().map(|h| h.serialize()).collect();
+    pub fn serialize_all(&self) -> Result<(Vec<u8>, Vec<usize>), FormatError> {
+        let serialized: Vec<Vec<u8>> = self
+            .headers
+            .iter()
+            .map(|h| h.serialize())
+            .collect::<Result<_, _>>()?;
         let total: usize = serialized.iter().map(|s| s.len()).sum();
         let mut buf = Vec::with_capacity(total);
         let mut offsets = Vec::with_capacity(serialized.len());
@@ -141,7 +169,7 @@ impl BatchObjectHeaderWriter {
             offsets.push(buf.len());
             buf.extend_from_slice(s);
         }
-        (buf, offsets)
+        Ok((buf, offsets))
     }
 }
 
@@ -159,7 +187,7 @@ mod tests {
     #[test]
     fn empty_header_roundtrip() {
         let writer = ObjectHeaderWriter::new();
-        let bytes = writer.serialize();
+        let bytes = writer.serialize().unwrap();
         let hdr = ObjectHeader::parse(&bytes, 0, 8, 8).unwrap();
         assert_eq!(hdr.version, 2);
         assert_eq!(hdr.messages.len(), 0);
@@ -170,7 +198,7 @@ mod tests {
         let mut writer = ObjectHeaderWriter::new();
         writer.add_message(MessageType::Dataspace, vec![1, 2, 3, 4]);
         writer.add_message(MessageType::Datatype, vec![5, 6]);
-        let bytes = writer.serialize();
+        let bytes = writer.serialize().unwrap();
         let hdr = ObjectHeader::parse(&bytes, 0, 8, 8).unwrap();
         assert_eq!(hdr.messages.len(), 2);
         assert_eq!(hdr.messages[0].msg_type, MessageType::Dataspace);
@@ -184,12 +212,30 @@ mod tests {
         let mut writer = ObjectHeaderWriter::new();
         // Add a message with >255 bytes of payload
         writer.add_message(MessageType::Datatype, vec![0xAA; 300]);
-        let bytes = writer.serialize();
+        let bytes = writer.serialize().unwrap();
         let hdr = ObjectHeader::parse(&bytes, 0, 8, 8).unwrap();
         assert_eq!(hdr.messages.len(), 1);
         assert_eq!(hdr.messages[0].data.len(), 300);
     }
 
+    #[test]
+    fn oversized_message_is_an_error_not_a_truncated_size() {
+        // 65535 bytes is the largest encodable payload.
+        let mut writer = ObjectHeaderWriter::new();
+        writer.add_message(MessageType::Attribute, vec![0; MAX_MESSAGE_SIZE]);
+        let bytes = writer.serialize().unwrap();
+        let hdr = ObjectHeader::parse(&bytes, 0, 8, 8).unwrap();
+        assert_eq!(hdr.messages[0].data.len(), MAX_MESSAGE_SIZE);
+
+        // One byte more used to be written with its size wrapped to 0.
+        let mut writer = ObjectHeaderWriter::new();
+        writer.add_message(MessageType::Attribute, vec![0; MAX_MESSAGE_SIZE + 1]);
+        assert!(matches!(
+            writer.serialize(),
+            Err(FormatError::SerializationError(_))
+        ));
+    }
+
     #[test]
     fn batch_writer_serialize_all() {
         let mut batch = BatchObjectHeaderWriter::new();
@@ -204,7 +250,7 @@ mod tests {
         batch.add(w2);
         assert_eq!(batch.len(), 2);
 
-        let (buf, offsets) = batch.serialize_all();
+        let (buf, offsets) = batch.serialize_all().unwrap();
         assert_eq!(offsets.len(), 2);
         assert_eq!(offsets[0], 0);
 
@@ -222,7 +268,7 @@ mod tests {
     fn batch_writer_empty() {
         let batch = BatchObjectHeaderWriter::new();
         assert!(batch.is_empty());
-        let (buf, offsets) = batch.serialize_all();
+        let (buf, offsets) = batch.serialize_all().unwrap();
         assert!(buf.is_empty());
         assert!(offsets.is_empty());
     }

crates/clawhdf5-format/src/parallel_read.rs

@@ -10,7 +10,7 @@
 use crate::chunked_read::ChunkInfo;
 use crate::error::FormatError;
 use crate::filter_pipeline::FilterPipeline;
-use crate::filters::decompress_chunk;
+use crate::filters::decompress_chunk_masked;
 use crate::lane_partition::{self, LaneStats, PartitionStats};
 
 /// Threshold: only use parallel decompression when chunk count exceeds this.
@@ -84,11 +84,13 @@ pub fn decompress_chunks_lane_partitioned(
                 }
                 let raw_chunk = &file_data[c_addr..c_addr + size];
 
-                let decompressed = if chunk_info.filter_mask == 0 {
-                    decompress_chunk(raw_chunk, pipeline, chunk_total_bytes, element_size)?
-                } else {
-                    raw_chunk.to_vec()
-                };
+                let decompressed = decompress_chunk_masked(
+                    raw_chunk,
+                    pipeline,
+                    chunk_total_bytes,
+                    element_size,
+                    chunk_info.filter_mask,
+                )?;
 
                 stats.chunks_processed += 1;
                 stats.compressed_bytes += size as u64;
@@ -158,11 +160,13 @@ pub fn decompress_chunks_parallel(
             }
             let raw_chunk = &file_data[c_addr..c_addr + size];
 
-            let decompressed = if chunk_info.filter_mask == 0 {
-                decompress_chunk(raw_chunk, pipeline, chunk_total_bytes, element_size)?
-            } else {
-                raw_chunk.to_vec()
-            };
+            let decompressed = decompress_chunk_masked(
+                raw_chunk,
+                pipeline,
+                chunk_total_bytes,
+                element_size,
+                chunk_info.filter_mask,
+            )?;
 
             Ok(DecompressedChunk {
                 index,
@@ -200,11 +204,13 @@ pub fn decompress_chunks_sequential(
         let raw_chunk = &file_data[c_addr..c_addr + size];
 
         let decompressed = if let Some(pl) = pipeline {
-            if chunk_info.filter_mask == 0 {
-                decompress_chunk(raw_chunk, pl, chunk_total_bytes, element_size)?
-            } else {
-                raw_chunk.to_vec()
-            }
+            decompress_chunk_masked(
+                raw_chunk,
+                pl,
+                chunk_total_bytes,
+                element_size,
+                chunk_info.filter_mask,
+            )?
         } else {
             raw_chunk.to_vec()
         };

crates/clawhdf5-format/src/partial_read.rs

@@ -22,7 +22,7 @@ use crate::data_read::extract_selection_from_buffer;
 use crate::dataspace::Dataspace;
 use crate::error::FormatError;
 use crate::filter_pipeline::FilterPipeline;
-use crate::filters::decompress_chunk;
+use crate::filters::{all_filters_skipped, decompress_chunk_masked};
 use crate::selection::Selection;
 
 /// The smallest axis-aligned box containing every selected element, as
@@ -325,12 +325,18 @@ pub fn read_selection(
                         expected: at.saturating_add(chunk.chunk_size as usize),
                         available: file_data.len(),
                     })?;
-                // Mirrors the full-read path: a non-zero filter mask means the
-                // chunk was stored unfiltered.
+                // Mirrors the full-read path: filter-mask bit i set means
+                // filter i was not applied to this chunk.
                 let decoded;
                 let data: &[u8] = match pipeline {
-                    Some(pl) if chunk.filter_mask == 0 => {
-                        decoded = decompress_chunk(raw, pl, chunk_bytes, elem_size as u32)?;
+                    Some(pl) if !all_filters_skipped(pl, chunk.filter_mask) => {
+                        decoded = decompress_chunk_masked(
+                            raw,
+                            pl,
+                            chunk_bytes,
+                            elem_size as u32,
+                            chunk.filter_mask,
+                        )?;
                         &decoded
                     }
                     _ => raw,

crates/clawhdf5-format/src/property_list.rs

@@ -43,7 +43,7 @@ impl Default for DatasetCreateProps {
             fletcher32: false,
             lz4: false,
             zstd_level: None,
-            fill_time: FillTime::Alloc,
+            fill_time: FillTime::IfSet,
             compact: false,
             alignment: 0,
         }
@@ -335,7 +335,7 @@ mod tests {
     fn dcpl_defaults() {
         let dcpl = DatasetCreateProps::new();
         assert!(dcpl.chunk_dims.is_none());
-        assert_eq!(dcpl.fill_time, FillTime::Alloc);
+        assert_eq!(dcpl.fill_time, FillTime::IfSet);
         assert!(!dcpl.compact);
     }
 

crates/clawhdf5-format/src/shared_message.rs

@@ -225,9 +225,12 @@ pub fn parse_sohm_table_message(
 
 /// Parse the SOHM table structure (signature "SMTB") from the file.
 ///
-/// Each index entry: index_type(1) + mesg_types(2) + min_mesg_size(4) +
-///   list_max(2) + btree_min(2) + num_messages(2) + index_addr(offset_size) +
-///   heap_addr(offset_size)
+/// Each index entry: version(1) + index_type(1) + mesg_types(2) +
+///   min_mesg_size(4) + list_max(2) + btree_min(2) + num_messages(2) +
+///   index_addr(offset_size) + heap_addr(offset_size)
+///
+/// The leading per-index version byte (0) was missing here, so every field
+/// after it was read one byte off — verified against an HDF5 2.0 file.
 pub fn parse_sohm_table(
     file_data: &[u8],
     table_addr: usize,
@@ -240,11 +243,16 @@ pub fn parse_sohm_table(
     }
     let mut pos = table_addr + 4;
     let os = offset_size as usize;
-    let entry_size = 1 + 2 + 4 + 2 + 2 + 2 + os + os; // 13 + 2*offset_size
+    let entry_size = 1 + 1 + 2 + 4 + 2 + 2 + 2 + os + os; // 14 + 2*offset_size
 
     let mut indexes = Vec::with_capacity(nindexes as usize);
     for _ in 0..nindexes {
         ensure_len(file_data, pos, entry_size)?;
+        let version = file_data[pos];
+        if version != 0 {
+            return Err(FormatError::InvalidSohmTableVersion(version));
+        }
+        pos += 1;
         let index_type = file_data[pos];
         pos += 1;
         let mesg_types = u16::from_le_bytes([file_data[pos], file_data[pos + 1]]);
@@ -381,6 +389,68 @@ pub fn parse_sohm_btree_entries(
 // ---- SOHM resolution ----
 
 /// Find the SOHM index that handles the given message type.
+/// Load a file's SOHM table: superblock → superblock extension → Shared
+/// Message Table message → SMTB. `Ok(None)` when the file has no superblock
+/// extension or no shared-message table.
+pub fn load_sohm_table(
+    file_data: &[u8],
+    offset_size: u8,
+    length_size: u8,
+) -> Result<Option<SohmTable>, FormatError> {
+    let sig = crate::signature::find_signature(file_data)?;
+    let sb = crate::superblock::Superblock::parse(file_data, sig)?;
+    let Some(ext_addr) = sb
+        .superblock_extension_address
+        .filter(|&a| !is_undefined(a, offset_size))
+    else {
+        return Ok(None);
+    };
+    let ext = ObjectHeader::parse(file_data, ext_addr as usize, offset_size, length_size)?;
+    let Some(msg) = ext
+        .messages
+        .iter()
+        .find(|m| m.msg_type == MessageType::SharedMessageTable)
+    else {
+        return Ok(None);
+    };
+    let table_msg = parse_sohm_table_message(&msg.data, offset_size)?;
+    parse_sohm_table(
+        file_data,
+        table_msg.table_address as usize,
+        table_msg.nindexes,
+        offset_size,
+    )
+    .map(Some)
+}
+
+/// Like [`message_data`], but also follows references into the file's SOHM
+/// heap (shared object header messages), loading the SOHM table on demand.
+pub fn message_data_with_sohm<'a>(
+    file_data: &[u8],
+    msg: &'a crate::object_header::HeaderMessage,
+    offset_size: u8,
+    length_size: u8,
+) -> Result<Cow<'a, [u8]>, FormatError> {
+    if !is_shared(msg.flags) {
+        return Ok(Cow::Borrowed(&msg.data));
+    }
+    let shared_ref = parse_shared_ref(&msg.data, offset_size)?;
+    let table = if shared_ref.heap_id.is_some() {
+        load_sohm_table(file_data, offset_size, length_size)?
+    } else {
+        None
+    };
+    resolve_shared_message_with_sohm(
+        file_data,
+        &shared_ref,
+        msg.msg_type,
+        offset_size,
+        length_size,
+        table.as_ref(),
+    )
+    .map(Cow::Owned)
+}
+
 fn find_index_for_msg_type(table: &SohmTable, msg_type: MessageType) -> Option<&SohmIndex> {
     let type_bit = 1u16 << msg_type.to_u16();
     table
@@ -707,6 +777,7 @@ mod tests {
         let mut buf = Vec::new();
         buf.extend_from_slice(b"SMTB");
         for idx in indexes {
+            buf.push(0); // version
             buf.push(idx.index_type);
             buf.extend_from_slice(&idx.mesg_types.to_le_bytes());
             buf.extend_from_slice(&idx.min_mesg_size.to_le_bytes());

crates/clawhdf5-format/src/superblock.rs

@@ -39,7 +39,13 @@ pub struct Superblock {
     pub superblock_extension_address: Option<u64>,
     /// CRC32C checksum (v2/v3 only).
     pub checksum: Option<u32>,
-    /// Page size for page-buffer mode (v4 only). `None` for v0–v3.
+    /// Page size of the non-standard "version 4" superblock layout (v4 only).
+    /// `None` for v0–v3.
+    ///
+    /// HDF5 has no superblock version 4 — libhdf5 refuses it. A real paged
+    /// file is a v2/v3 superblock whose extension holds a File Space Info
+    /// message (what `FileWriter::with_page_size` writes). This field is kept
+    /// only so such files written by older clawhdf5 versions still parse.
     pub page_size: Option<u32>,
 }
 
@@ -127,8 +133,9 @@ impl Superblock {
 
     /// Serialize this superblock to bytes.
     ///
-    /// Writes v2/v3 format, or v4 (with `page_size`) when `self.version == 4`.
-    /// Computes and appends Jenkins lookup3 checksum.
+    /// Writes v2/v3 format, or the non-standard v4 (with `page_size`) when
+    /// `self.version == 4` — which no HDF5 library opens; see
+    /// [`Self::page_size`]. Computes and appends Jenkins lookup3 checksum.
     pub fn serialize(&self) -> Vec<u8> {
         let mut buf = Vec::with_capacity(48);
         buf.extend_from_slice(&HDF5_SIGNATURE);

crates/clawhdf5-format/src/type_builders.rs

@@ -15,31 +15,83 @@ use crate::datatype::{
 
 /// Controls when fill values are written to dataset storage.
 ///
-/// Corresponds to the HDF5 fill value message's "fill time" field.
+/// Corresponds to the HDF5 fill value message's "fill time" field
+/// (`H5D_fill_time_t`).
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
 pub enum FillTime {
-    /// Never write fill values (0x02). Avoids initialization overhead
-    /// for datasets that will be fully written before any read.
+    /// Never write fill values (`H5D_FILL_TIME_NEVER`). Avoids
+    /// initialization overhead for datasets that will be fully written
+    /// before any read.
     Never,
-    /// Write fill values at allocation time (0x0a). This is the default
-    /// and matches the HDF5 C library's behavior.
-    #[default]
+    /// Write fill values when storage is allocated (`H5D_FILL_TIME_ALLOC`).
     Alloc,
-    /// Write fill values only when the fill value has been explicitly set (0x06).
+    /// Write fill values at allocation only if one was set explicitly
+    /// (`H5D_FILL_TIME_IFSET`). The default, as in the HDF5 C library.
+    #[default]
     IfSet,
 }
 
+/// Space allocation time written with every fill value message: late
+/// (`H5D_ALLOC_TIME_LATE`), bits 0-1 of the flags byte.
+const ALLOC_TIME_LATE: u8 = 2;
+
 impl FillTime {
-    /// Serialize to the byte used in the fill value message (version 3).
+    /// Serialize to the flags byte of a version 3 fill value message: the
+    /// space allocation time (late) in bits 0-1 and the fill time in bits
+    /// 2-3 (`H5D_FILL_TIME_ALLOC` = 0, `NEVER` = 1, `IFSET` = 2).
+    ///
+    /// This used to put `Never` in the ALLOC slot, `Alloc` in IFSET and
+    /// `IfSet` in NEVER, so libhdf5 saw every choice as a different one.
     pub fn to_byte(self) -> u8 {
+        ALLOC_TIME_LATE | (self.code() << 2)
+    }
+
+    /// Decode the fill time from a version 3 fill value message's flags.
+    pub fn from_byte(flags: u8) -> Option<FillTime> {
+        match (flags >> 2) & 0x03 {
+            0 => Some(FillTime::Alloc),
+            1 => Some(FillTime::Never),
+            2 => Some(FillTime::IfSet),
+            _ => None,
+        }
+    }
+
+    fn code(self) -> u8 {
         match self {
-            FillTime::Never => 0x02,
-            FillTime::Alloc => 0x0a,
-            FillTime::IfSet => 0x06,
+            FillTime::Alloc => 0,
+            FillTime::Never => 1,
+            FillTime::IfSet => 2,
         }
     }
 }
 
+/// Serialize a version 3 Fill Value message for a dataset of `dt`: the fill
+/// time, and the user-defined fill value if there is one (bit 5).
+pub(crate) fn fill_value_message(
+    fill_time: FillTime,
+    value: Option<&[u8]>,
+    dt: &Datatype,
+) -> Result<Vec<u8>, crate::error::FormatError> {
+    let mut msg = vec![3, fill_time.to_byte()];
+    if let Some(value) = value {
+        if matches!(dt, Datatype::VariableLength { .. }) {
+            return Err(crate::error::FormatError::SerializationError(
+                "a fill value for a variable-length datatype is not supported".into(),
+            ));
+        }
+        if value.len() != dt.type_size() as usize {
+            return Err(crate::error::FormatError::DataSizeMismatch {
+                expected: dt.type_size() as usize,
+                actual: value.len(),
+            });
+        }
+        msg[1] |= 0x20; // fill value defined
+        msg.extend_from_slice(&(value.len() as u32).to_le_bytes());
+        msg.extend_from_slice(value);
+    }
+    Ok(msg)
+}
+
 // ---- Datatype constructors ----
 
 pub fn make_f64_type() -> Datatype {
@@ -56,6 +108,21 @@ pub fn make_f64_type() -> Datatype {
     }
 }
 
+/// IEEE-754 half precision (binary16), little-endian — numpy's `float16`.
+pub fn make_f16_type() -> Datatype {
+    Datatype::FloatingPoint {
+        size: 2,
+        byte_order: DatatypeByteOrder::LittleEndian,
+        bit_offset: 0,
+        bit_precision: 16,
+        exponent_location: 10,
+        exponent_size: 5,
+        mantissa_location: 0,
+        mantissa_size: 10,
+        exponent_bias: 15,
+    }
+}
+
 pub fn make_f32_type() -> Datatype {
     Datatype::FloatingPoint {
         size: 4,
@@ -317,7 +384,11 @@ pub(crate) fn build_attr_message(name: &str, value: &AttrValue) -> AttributeMess
             raw_data: data.clone(),
         },
         AttrValue::String(s) => {
-            let bytes = s.as_bytes();
+            // A fixed-length string type must be at least 1 byte: libhdf5
+            // rejects size 0 ("invalid datatype size") and with it every
+            // attribute on the object. h5py stores "" as one NUL byte.
+            let mut bytes = s.as_bytes().to_vec();
+            bytes.resize(bytes.len().max(1), 0);
             AttributeMessage {
                 name: name.to_string(),
                 datatype: Datatype::String {
@@ -326,11 +397,12 @@ pub(crate) fn build_attr_message(name: &str, value: &AttrValue) -> AttributeMess
                     charset: CharacterSet::Utf8,
                 },
                 dataspace: scalar_ds(),
-                raw_data: bytes.to_vec(),
+                raw_data: bytes,
             }
         }
         AttrValue::StringArray(arr) => {
-            let max_len = arr.iter().map(|s| s.len()).max().unwrap_or(0);
+            // At least 1 byte per element, as for a single string.
+            let max_len = arr.iter().map(|s| s.len()).max().unwrap_or(0).max(1);
             let mut raw = Vec::new();
             for s in arr {
                 let mut b = s.as_bytes().to_vec();
@@ -416,8 +488,10 @@ pub struct DatasetBuilder {
     pub(crate) data: Option<Vec<u8>>,
     pub(crate) attrs: Vec<(String, AttrValue)>,
     pub(crate) chunk_options: ChunkOptions,
-    /// Controls when fill values are written. Default is `FillTime::Alloc`.
+    /// Controls when fill values are written. Default is `FillTime::IfSet`.
     pub(crate) fill_time: FillTime,
+    /// User-defined fill value: one element's bytes, as stored.
+    pub(crate) fill_value: Option<Vec<u8>>,
     /// Use compact (inline) storage: data is stored in the object header.
     /// Only valid when raw data is <= 65536 bytes and dataset is not chunked.
     pub(crate) compact: bool,
@@ -444,6 +518,7 @@ impl DatasetBuilder {
             attrs: Vec::new(),
             chunk_options: ChunkOptions::default(),
             fill_time: FillTime::default(),
+            fill_value: None,
             compact: false,
             alignment: 0,
             virtual_sources: None,
@@ -478,6 +553,24 @@ impl DatasetBuilder {
         self
     }
 
+    /// Store `data` as IEEE half precision (numpy `float16`), rounding each
+    /// value to the nearest half ([`crate::float16::f32_to_f16_bits`]).
+    /// Half the bytes of [`Self::with_f32_data`], at about three significant
+    /// decimal digits; values beyond ±65504 become ±infinity. Reading it back
+    /// with `read_f32` yields the rounded values exactly.
+    pub fn with_f16_data(&mut self, data: &[f32]) -> &mut Self {
+        self.datatype = Some(make_f16_type());
+        let mut b = Vec::with_capacity(data.len() * 2);
+        for &v in data {
+            b.extend_from_slice(&crate::float16::f32_to_f16_bits(v).to_le_bytes());
+        }
+        self.data = Some(b);
+        if self.shape.is_none() {
+            self.shape = Some(vec![data.len() as u64]);
+        }
+        self
+    }
+
     pub fn with_i32_data(&mut self, data: &[i32]) -> &mut Self {
         self.datatype = Some(make_i32_type());
         let mut b = Vec::with_capacity(data.len() * 4);
@@ -638,7 +731,12 @@ impl DatasetBuilder {
         self
     }
 
-    /// Enable Pcodec lossless numerical compression (clawhdf5 filter ID 32023).
+    /// Enable Pcodec lossless numerical compression (private clawhdf5 filter
+    /// ID 480).
+    ///
+    /// **Not interoperable:** pcodec has no registered HDF5 filter ID and no
+    /// libhdf5 plugin, so h5py and other HDF5 readers cannot read the
+    /// dataset — only clawhdf5 built with the `pcodec` feature can.
     ///
     /// Pcodec achieves 30–94% better compression ratio than Zstd for f32/f64
     /// columns at 1–5 GiB/s decompression speed (arXiv:2502.06112). Requires
@@ -682,10 +780,20 @@ impl DatasetBuilder {
         self
     }
 
+    /// Set the dataset's fill value: what readers return for storage that
+    /// was never written (e.g. after the dataset is extended). `value` is one
+    /// element's bytes as stored — the dataset datatype's size and byte order
+    /// (`(-1i32).to_le_bytes()` for an `i32` dataset). A size mismatch, or a
+    /// variable-length datatype, makes `finish` fail.
+    pub fn with_fill_value(&mut self, value: &[u8]) -> &mut Self {
+        self.fill_value = Some(value.to_vec());
+        self
+    }
+
     /// Use compact (inline) storage for this dataset.
     ///
     /// The raw data is stored directly in the dataset's object header rather
-    /// than as a separate data blob. Only effective when raw data <= 65536 bytes
+    /// than as a separate data blob. Only effective when raw data <= 65531 bytes
     /// and the dataset is not chunked.
     pub fn compact(&mut self) -> &mut Self {
         self.compact = true;

crates/clawhdf5-format/src/vl_data.rs

@@ -148,7 +148,12 @@ pub fn read_vl_strings(
     Ok(result)
 }
 
-/// Resolve VL byte sequences from raw data.
+/// Resolve VL sequences from raw data, returning each element's bytes.
+///
+/// Each element is the sequence's full encoding — element count × base type
+/// size bytes, in the base type's byte order — so a sequence of `i32` yields
+/// four bytes per value. Decode it with the base type (e.g.
+/// [`crate::data_read::read_as_i64`]).
 pub fn read_vl_bytes(
     file_data: &[u8],
     raw_data: &[u8],
@@ -177,8 +182,10 @@ pub fn read_vl_bytes(
             },
         )?;
 
-        let len = (vl.length as usize).min(obj.data.len());
-        result.push(obj.data[..len].to_vec());
+        // The heap object holds the whole sequence. `vl.length` counts
+        // elements, not bytes, so it is only the byte length when the base
+        // type is one byte wide.
+        result.push(obj.data.clone());
     }
 
     Ok(result)

crates/clawhdf5-format/tests/fixtures/filters/README.md

@@ -0,0 +1,13 @@
+# Filter conformance fixtures
+
+Files written by libhdf5 (and its registered filter plugins), used by the
+filter regression tests in `src/filters.rs` to compare our decoders against
+the values h5py/libhdf5 read from the same bytes. Chunk byte ranges quoted in
+the tests come from h5py's `DatasetID.get_chunk_info`.
+
+| File | Origin | Licence |
+|------|--------|---------|
+| `h5ex_d_lz4.h5` | HDF Group `HDF5Examples/C/H5FLT/tfiles/h5ex_d_lz4.h5` (hdf5 repository) | HDF5 licence (BSD-3-Clause style) |
+| `noencoder.h5` | HDF Group `test/testfiles/noencoder.h5` (hdf5 repository) | HDF5 licence (BSD-3-Clause style) |
+| `le_data.h5` | HDF Group `test/testfiles/le_data.h5` (hdf5 repository) | HDF5 licence (BSD-3-Clause style) |
+| `szip_h5py.h5` | Written for these tests with h5py 3 / libhdf5 2.0.0 (libaec szip): `f8` (8x10, chunks 4x10, `('nn', 8)`), `i8` (8x10, chunks 4x10, `('ec', 4)`), `u2` (70, chunks 35, `('nn', 8)`) | Same as this repository |

crates/clawhdf5-format/tests/fixtures/gen_shared_fill.py

@@ -0,0 +1,49 @@
+"""Generate shared_fill_value.h5: datasets whose Fill Value message is
+*shared*, in the two ways libhdf5 can share one.
+
+- /sohm_a, /sohm_b: the file has a shared-object-header-message (SOHM) index
+  for fill values, so libhdf5 stores the fill value (-7, int32) in the SOHM
+  heap and /sohm_b's header holds only a reference to it. Chunked, with only
+  the first chunk written, so the rest reads as the fill value.
+- /unwritten_a, /unwritten_b: the same, never written: no storage at all,
+  read entirely as the fill value.
+
+h5py has no API for SOHM indexes, so the file creation property list is
+configured by calling the libhdf5 bundled in the h5py wheel through ctypes.
+Written with h5py 3.16.0 / HDF5 2.0.0. Re-run only to regenerate:
+
+    python gen_shared_fill.py shared_fill_value.h5
+"""
+import ctypes
+import glob
+import os
+import sys
+
+import h5py
+import numpy as np
+
+libdir = os.path.join(os.path.dirname(os.path.dirname(h5py.__file__)), "h5py.libs")
+libs = [p for p in glob.glob(os.path.join(libdir, "libhdf5*.so*")) if "_hl" not in os.path.basename(p)]
+lib = ctypes.CDLL(libs[0])
+lib.H5open()
+
+H5O_SHMESG_FILL_FLAG = 1 << 0x0005
+
+fcpl = h5py.h5p.create(h5py.h5p.FILE_CREATE)
+lib.H5Pset_shared_mesg_nindexes.argtypes = [ctypes.c_int64, ctypes.c_uint]
+lib.H5Pset_shared_mesg_index.argtypes = [ctypes.c_int64, ctypes.c_uint, ctypes.c_uint, ctypes.c_uint]
+assert lib.H5Pset_shared_mesg_nindexes(fcpl.id, 1) >= 0
+assert lib.H5Pset_shared_mesg_index(fcpl.id, 0, H5O_SHMESG_FILL_FLAG, 0) >= 0
+
+fapl = h5py.h5p.create(h5py.h5p.FILE_ACCESS)
+fapl.set_libver_bounds(h5py.h5f.LIBVER_LATEST, h5py.h5f.LIBVER_LATEST)
+fid = h5py.h5f.create(sys.argv[1].encode(), h5py.h5f.ACC_TRUNC, fcpl=fcpl, fapl=fapl)
+with h5py.File(fid) as f:
+    # Chunked, with only the first chunk written: the rest reads as fill.
+    # libhdf5 keeps the first copy of a message in its own header; the second
+    # identical one (the `_b` datasets) is the SOHM reference.
+    for name in ("sohm_a", "sohm_b"):
+        d = f.create_dataset(name, shape=(8,), chunks=(4,), dtype="<i4", fillvalue=-7)
+        d[:4] = np.arange(4)
+    for name in ("unwritten_a", "unwritten_b"):
+        f.create_dataset(name, shape=(3,), dtype="<i4", fillvalue=-7)

crates/clawhdf5-format/tests/robustness_tests.rs

@@ -312,3 +312,49 @@ fn provenance_mismatch_on_corruption() {
         "corrupted data should produce hash mismatch"
     );
 }
+
+// ---------------------------------------------------------------------------
+// Fuzzer finds, kept as regression tests
+// ---------------------------------------------------------------------------
+
+/// `fuzz_btree_v2` crash input from 2026-09-20 (82 bytes): a B-tree v2 header
+/// followed by internal nodes that point back into themselves. It predates the
+/// depth cap and record budget added to B-tree v2 traversal that day and no
+/// longer crashes; this replays the fuzz target's exact code path on it so a
+/// regression fails CI rather than waiting for a fuzz run.
+#[test]
+fn fuzz_btree_v2_crash_f98c19dc_is_a_clean_result() {
+    use clawhdf5_format::btree_v2::{BTreeV2Header, collect_btree_v2_records};
+    let data: &[u8] = &[
+        0x42, 0x54, 0x48, 0x44, 0x00, 0x06, 0x00, 0xed, 0xef, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00,
+        0x00, 0x03, 0x40, 0x14, 0x93, 0x42, 0x54, 0x49, 0x4e, 0x42, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x42, 0x54, 0x48, 0x44, 0x00, 0x00, 0x00, 0x13, 0x05, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x40, 0x14, 0x93, 0x42, 0x54, 0x00, 0x49,
+        0x00, 0x01, 0x4e, 0x42, 0x42, 0x54, 0xbe,
+    ];
+    assert_eq!(data.len(), 82);
+    for offset_size in [4u8, 8] {
+        for length_size in [4u8, 8] {
+            if let Ok(header) = BTreeV2Header::parse(data, 0, offset_size, length_size) {
+                let _ = collect_btree_v2_records(data, &header, offset_size, length_size);
+            }
+        }
+    }
+    let (fields, file) = data.split_first_chunk::<20>().unwrap();
+    let header = BTreeV2Header {
+        tree_type: fields[0],
+        node_size: u32::from_le_bytes([fields[1], fields[2], fields[3], fields[4]]),
+        record_size: u16::from_le_bytes([fields[5], fields[6]]),
+        depth: u16::from_le_bytes([fields[7], fields[8]]),
+        root_node_address: u64::from(u32::from_le_bytes([
+            fields[9], fields[10], fields[11], fields[12],
+        ])),
+        num_records_in_root: u16::from_le_bytes([fields[13], fields[14]]),
+        total_records: u64::from(u32::from_le_bytes([
+            fields[15], fields[16], fields[17], fields[18],
+        ])),
+    };
+    let offset_size = if fields[19] & 1 == 0 { 4 } else { 8 };
+    let _ = collect_btree_v2_records(file, &header, offset_size, 8);
+}

crates/clawhdf5-format/tests/writer_h5py_tests.rs

@@ -940,3 +940,61 @@ fn provenance_verify_written_file() {
             .unwrap();
     assert_eq!(result, clawhdf5_format::provenance::VerifyResult::Ok);
 }
+
+// ---- hdf5plugin interop: registered third-party compression filters ----
+
+/// Write `data` (f64, 1-D, chunked) with `configure` applied, then read it
+/// back with h5py + hdf5plugin (libhdf5's registered filter plugins) and
+/// return the values it decodes.
+#[cfg(any(feature = "lz4", feature = "zstd"))]
+fn hdf5plugin_roundtrip(
+    tag: &str,
+    data: &[f64],
+    configure: impl FnOnce(&mut clawhdf5_format::type_builders::DatasetBuilder),
+) -> Vec<f64> {
+    let mut fw = FileWriter::new();
+    let ds = fw.create_dataset("data");
+    ds.with_f64_data(data)
+        .with_shape(&[data.len() as u64])
+        .with_chunks(&[250]);
+    configure(ds);
+    let bytes = fw.finish().unwrap();
+    let path = std::env::temp_dir().join(format!("clawhdf5_hdf5plugin_{tag}.h5"));
+    std::fs::write(&path, &bytes).unwrap();
+    let script = format!(
+        "import h5py,hdf5plugin,json; f=h5py.File('{}','r'); print(json.dumps(f['data'][:].tolist()))",
+        path.display()
+    );
+    let stdout = h5py_read(&path, &script);
+    serde_json::from_str(&stdout).unwrap()
+}
+
+/// libhdf5's LZ4 plugin must decode what we write (it could not while we
+/// wrote a private 4-byte-LE-size framing).
+#[cfg(feature = "lz4")]
+#[test]
+#[ignore = "requires Python h5py + hdf5plugin"]
+fn hdf5plugin_reads_our_lz4() {
+    let data: Vec<f64> = (0..1000).map(|i| (i % 37) as f64 * 0.5).collect();
+    let got = hdf5plugin_roundtrip("lz4", &data, |ds| {
+        ds.with_lz4();
+    });
+    assert_eq!(got, data);
+    let got = hdf5plugin_roundtrip("lz4_noshuffle", &data, |ds| {
+        ds.with_lz4().without_shuffle();
+    });
+    assert_eq!(got, data);
+}
+
+/// libhdf5's Zstandard plugin must decode what we write (it could not while
+/// our frames lacked the content size).
+#[cfg(feature = "zstd")]
+#[test]
+#[ignore = "requires Python h5py + hdf5plugin"]
+fn hdf5plugin_reads_our_zstd() {
+    let data: Vec<f64> = (0..1000).map(|i| (i % 37) as f64 * 0.5).collect();
+    let got = hdf5plugin_roundtrip("zstd", &data, |ds| {
+        ds.with_zstd(3);
+    });
+    assert_eq!(got, data);
+}

crates/clawhdf5-format/tests/writer_meta_tests.rs

@@ -0,0 +1,646 @@
+//! Regression tests for writer metadata bugs that produced files libhdf5
+//! refuses (or reads differently from us), plus the reader-side counterparts.
+//!
+//! The plain tests check the bytes we write with our own parser. The
+//! `#[ignore]`d ones are the interop half: they open what we write in h5py
+//! (`CLAWHDF5_PYTHON`, as in `writer_h5py_tests.rs`) and run `h5dump` over it.
+
+use clawhdf5_format::data_layout::DataLayout;
+use clawhdf5_format::datatype::{Datatype, DatatypeByteOrder, ReferenceType};
+use clawhdf5_format::file_writer::{AttrValue, FileWriter};
+use clawhdf5_format::group_v2::resolve_path_any;
+use clawhdf5_format::message_type::MessageType;
+use clawhdf5_format::object_header::ObjectHeader;
+use clawhdf5_format::signature;
+use clawhdf5_format::superblock::Superblock;
+use clawhdf5_format::type_builders::{FillTime, make_u8_type};
+
+// ---- helpers ----
+
+fn header_at(bytes: &[u8], path: &str) -> (Superblock, ObjectHeader) {
+    let sig = signature::find_signature(bytes).unwrap();
+    let sb = Superblock::parse(bytes, sig).unwrap();
+    let addr = if path == "/" {
+        sb.root_group_address
+    } else {
+        resolve_path_any(bytes, &sb, path).unwrap()
+    };
+    let oh = ObjectHeader::parse(bytes, addr as usize, sb.offset_size, sb.length_size).unwrap();
+    (sb, oh)
+}
+
+fn layout_of(bytes: &[u8], path: &str) -> DataLayout {
+    let (sb, oh) = header_at(bytes, path);
+    let msg = oh
+        .messages
+        .iter()
+        .find(|m| m.msg_type == MessageType::DataLayout)
+        .unwrap();
+    DataLayout::parse(&msg.data, sb.offset_size, sb.length_size).unwrap()
+}
+
+fn python() -> String {
+    std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
+}
+
+fn write_tmp(name: &str, bytes: &[u8]) -> std::path::PathBuf {
+    let path = std::env::temp_dir().join(format!("clawhdf5_writer_meta_{name}.h5"));
+    std::fs::write(&path, bytes).unwrap();
+    path
+}
+
+/// Run `script` (with `path` bound to the file) under h5py; return stdout.
+fn h5py(path: &std::path::Path, script: &str) -> String {
+    let full = format!(
+        "import h5py, numpy as np, json\npath = {:?}\n{script}",
+        path.display().to_string()
+    );
+    let o = std::process::Command::new(python())
+        .args(["-c", &full])
+        .output()
+        .expect("python interpreter");
+    assert!(
+        o.status.success(),
+        "h5py failed: {}",
+        String::from_utf8_lossy(&o.stderr)
+    );
+    String::from_utf8(o.stdout).unwrap().trim().to_string()
+}
+
+/// `h5dump` must read the whole file without error.
+fn h5dump_ok(path: &std::path::Path) {
+    let o = std::process::Command::new("h5dump")
+        .arg(path)
+        .output()
+        .expect("h5dump");
+    assert!(
+        o.status.success(),
+        "h5dump failed: {}{}",
+        String::from_utf8_lossy(&o.stdout),
+        String::from_utf8_lossy(&o.stderr)
+    );
+}
+
+fn u8_ramp(n: usize) -> Vec<u8> {
+    (0..n).map(|i| (i % 251) as u8).collect()
+}
+
+// ---- 1. object header message size limit ----
+
+#[test]
+fn attribute_too_big_for_a_header_message_is_an_error() {
+    // Measured: a 70000-byte attribute was written with its message size
+    // wrapped to 16 bits, and libhdf5 refused the whole root group.
+    let mut fw = FileWriter::new();
+    fw.set_root_attr(
+        "a",
+        AttrValue::Raw {
+            datatype: make_u8_type(),
+            shape: vec![70_000],
+            data: u8_ramp(70_000),
+        },
+    );
+    assert!(fw.finish().is_err());
+
+    // 65500 bytes still fits and still works.
+    let mut fw = FileWriter::new();
+    fw.set_root_attr(
+        "a",
+        AttrValue::Raw {
+            datatype: make_u8_type(),
+            shape: vec![65_500],
+            data: u8_ramp(65_500),
+        },
+    );
+    let bytes = fw.finish().unwrap();
+    let (sb, oh) = header_at(&bytes, "/");
+    let attrs = clawhdf5_format::attribute::extract_attributes(&oh, sb.length_size).unwrap();
+    assert_eq!(attrs[0].raw_data, u8_ramp(65_500));
+}
+
+#[test]
+fn compact_layout_falls_back_to_contiguous_past_the_message_limit() {
+    // Layout message = 4 bytes + data; data may be at most 65531 bytes.
+    for (n, compact) in [(65_531, true), (65_532, false), (65_534, false)] {
+        let mut fw = FileWriter::new();
+        fw.create_dataset("d").with_u8_data(&u8_ramp(n)).compact();
+        let bytes = fw.finish().unwrap();
+        match layout_of(&bytes, "d") {
+            DataLayout::Compact { data } => {
+                assert!(compact, "{n} bytes must not be compact");
+                assert_eq!(data, u8_ramp(n));
+            }
+            DataLayout::Contiguous { .. } => assert!(!compact, "{n} bytes should be compact"),
+            other => panic!("unexpected layout {other:?}"),
+        }
+    }
+}
+
+#[test]
+#[ignore = "requires Python h5py module and h5dump"]
+fn h5py_reads_compact_datasets_at_the_limit() {
+    for n in [65_531usize, 65_534] {
+        let mut fw = FileWriter::new();
+        fw.create_dataset("d").with_u8_data(&u8_ramp(n)).compact();
+        let path = write_tmp(&format!("compact_{n}"), &fw.finish().unwrap());
+        let out = h5py(
+            &path,
+            "f = h5py.File(path, 'r'); v = f['d'][()]\n\
+             print(bool((v == (np.arange(v.size) % 251).astype(np.uint8)).all()), v.size)",
+        );
+        assert_eq!(out, format!("True {n}"));
+        h5dump_ok(&path);
+    }
+}
+
+// ---- 2. Time / BitField / Opaque / Reference datatypes ----
+
+fn exotic_types() -> Vec<(&'static str, Datatype, Vec<u8>)> {
+    // Four elements each. The object references point at the root group,
+    // which a v3-superblock file without an extension puts at address 48.
+    let refs: Vec<u8> = (0..4).flat_map(|_| 48u64.to_le_bytes()).collect();
+    vec![
+        (
+            "bits",
+            Datatype::BitField {
+                size: 1,
+                byte_order: DatatypeByteOrder::LittleEndian,
+                bit_offset: 0,
+                bit_precision: 8,
+            },
+            vec![1, 2, 4, 8],
+        ),
+        (
+            "opaque",
+            Datatype::Opaque {
+                size: 4,
+                tag: b"mytag".to_vec(),
+            },
+            (0..16).collect(),
+        ),
+        (
+            "ref",
+            Datatype::Reference {
+                size: 8,
+                ref_type: ReferenceType::Object,
+            },
+            refs,
+        ),
+        (
+            "time",
+            Datatype::Time {
+                size: 4,
+                bit_precision: 32,
+            },
+            (0..16).collect(),
+        ),
+    ]
+}
+
+fn exotic_file() -> Vec<u8> {
+    let mut fw = FileWriter::new();
+    for (name, dt, raw) in exotic_types() {
+        fw.create_dataset(name)
+            .with_compound_data(dt.clone(), raw.clone(), 4);
+        fw.set_root_attr(
+            name,
+            AttrValue::Raw {
+                datatype: dt,
+                shape: vec![4],
+                data: raw,
+            },
+        );
+    }
+    fw.finish().unwrap()
+}
+
+#[test]
+fn exotic_datatypes_are_written_not_emptied() {
+    let bytes = exotic_file();
+    let (sb, root) = header_at(&bytes, "/");
+    assert_eq!(sb.root_group_address, 48);
+    let attrs = clawhdf5_format::attribute::extract_attributes(&root, sb.length_size).unwrap();
+    for (name, dt, raw) in exotic_types() {
+        let (_, oh) = header_at(&bytes, name);
+        let msg = oh
+            .messages
+            .iter()
+            .find(|m| m.msg_type == MessageType::Datatype)
+            .unwrap();
+        assert_eq!(msg.data, dt.serialize(), "{name}");
+        assert_eq!(Datatype::parse(&msg.data).unwrap().0, dt, "{name}");
+        let attr = attrs.iter().find(|a| a.name == name).unwrap();
+        assert_eq!(attr.datatype, dt, "{name}");
+        assert_eq!(attr.raw_data, raw, "{name}");
+    }
+}
+
+#[test]
+#[ignore = "requires Python h5py module and h5dump"]
+fn h5py_reads_exotic_datatypes() {
+    let path = write_tmp("exotic", &exotic_file());
+    let out = h5py(
+        &path,
+        "from h5py import h5t, h5s\n\
+         f = h5py.File(path, 'r')\n\
+         r = {}\n\
+         buf = np.zeros(4, dtype='V4')\n\
+         f['opaque'].id.read(h5s.ALL, h5s.ALL, buf, mtype=f['opaque'].id.get_type())\n\
+         r['bits'] = f['bits'][()].tolist(), f.attrs['bits'].tolist()\n\
+         r['opaque'] = (f['opaque'].id.get_type().get_tag().decode(),\n\
+         \x20              f.attrs.get_id('opaque').get_type().get_tag().decode(),\n\
+         \x20              buf.tobytes().hex())\n\
+         r['ref'] = [f[x].name for x in f['ref'][()]] + [f[x].name for x in f.attrs['ref']]\n\
+         r['time'] = (f['time'].id.get_type().get_class() == h5t.TIME,\n\
+         \x20            f.attrs.get_id('time').get_type().get_class() == h5t.TIME)\n\
+         print(json.dumps(r))",
+    );
+    let v: serde_json::Value = serde_json::from_str(&out).unwrap();
+    assert_eq!(v["bits"], serde_json::json!([[1, 2, 4, 8], [1, 2, 4, 8]]));
+    assert_eq!(
+        v["opaque"],
+        serde_json::json!(["mytag", "mytag", "000102030405060708090a0b0c0d0e0f"])
+    );
+    assert_eq!(v["ref"], serde_json::json!(vec!["/"; 8]));
+    assert_eq!(v["time"], serde_json::json!([true, true]));
+    h5dump_ok(&path);
+}
+
+#[test]
+#[ignore = "requires Python h5py module and h5dump"]
+fn raw_attributes_copied_from_h5py_survive_a_rewrite() {
+    // Read Raw attributes of the exotic classes out of an h5py file and write
+    // them back: this used to emit empty datatype messages.
+    let src = std::env::temp_dir().join("clawhdf5_writer_meta_exotic_src.h5");
+    h5py(
+        &src,
+        "from h5py import h5t, h5s, h5a\n\
+         f = h5py.File(path, 'w')\n\
+         f.attrs['ref'] = np.array([f.ref, f.ref], dtype=h5py.ref_dtype)\n\
+         f.attrs.create('opaque', np.frombuffer(b'abcdefgh', dtype='V4'))\n\
+         t = h5t.STD_B16BE.copy()\n\
+         a = h5a.create(f.id, b'bits', t, h5s.create_simple((2,)))\n\
+         a.write(np.array([0x0102, 0x0304], dtype='>u2'), mtype=t)\n\
+         a.close()\n\
+         f.close()",
+    );
+    let src_bytes = std::fs::read(&src).unwrap();
+    let (sb, root) = header_at(&src_bytes, "/");
+    let attrs = clawhdf5_format::attribute::extract_attributes(&root, sb.length_size).unwrap();
+    assert_eq!(attrs.len(), 3);
+    let mut fw = FileWriter::new();
+    for a in &attrs {
+        let data = if a.name == "ref" {
+            // Re-target the references at our root group.
+            48u64.to_le_bytes().repeat(2)
+        } else {
+            a.raw_data.clone()
+        };
+        fw.set_root_attr(
+            &a.name,
+            AttrValue::Raw {
+                datatype: a.datatype.clone(),
+                shape: a.dataspace.dimensions.clone(),
+                data,
+            },
+        );
+    }
+    let path = write_tmp("exotic_copy", &fw.finish().unwrap());
+    let out = h5py(
+        &path,
+        "f = h5py.File(path, 'r')\n\
+         print(json.dumps([[f[x].name for x in f.attrs['ref']],\n\
+         \x20                f.attrs['opaque'].tobytes().decode(),\n\
+         \x20                f.attrs.get_id('bits').get_type().get_order(),\n\
+         \x20                f.attrs['bits'].tolist()]))",
+    );
+    assert_eq!(out, r#"[["/", "/"], "abcdefgh", 1, [258, 772]]"#);
+    h5dump_ok(&path);
+}
+
+// ---- 3. paged file-space strategy ----
+
+fn paged_file(page_size: u32) -> Vec<u8> {
+    let mut fw = FileWriter::new();
+    fw.with_page_size(page_size);
+    fw.create_dataset("d").with_f64_data(&[1.0, 2.0, 3.0]);
+    fw.create_dataset("c")
+        .with_i32_data(&(0..100).collect::<Vec<_>>())
+        .with_chunks(&[10]);
+    fw.set_root_attr("a", AttrValue::I64(7));
+    let mut g = fw.create_group("g");
+    g.create_dataset("e").with_u8_data(&[9; 5000]);
+    fw.add_group(g.finish());
+    fw.finish().unwrap()
+}
+
+#[test]
+fn paged_file_has_a_real_superblock() {
+    // Measured: `with_page_size` wrote superblock version 4, which does not
+    // exist ("bad superblock version number" in libhdf5).
+    for ps in [512u32, 4096, 65536] {
+        let bytes = paged_file(ps);
+        let (sb, _) = header_at(&bytes, "/");
+        assert_eq!(sb.version, 3);
+        assert_eq!(bytes.len() % ps as usize, 0);
+        let (_, e) = header_at(&bytes, "g/e");
+        assert!(
+            e.messages
+                .iter()
+                .any(|m| m.msg_type == MessageType::Dataspace)
+        );
+    }
+}
+
+#[test]
+#[ignore = "requires Python h5py module and h5dump"]
+fn h5py_opens_paged_files() {
+    for ps in [512u32, 4096, 65536] {
+        let path = write_tmp(&format!("paged_{ps}"), &paged_file(ps));
+        let out = h5py(
+            &path,
+            "f = h5py.File(path, 'r')\n\
+             p = f.id.get_create_plist()\n\
+             print(json.dumps([p.get_file_space_strategy()[0], p.get_file_space_page_size(),\n\
+             \x20   f['d'][()].tolist(), int(f['c'][()].sum()), int(f.attrs['a']),\n\
+             \x20   int(f['g/e'][()].sum())]))",
+        );
+        assert_eq!(
+            out,
+            format!("[1, {ps}, [1.0, 2.0, 3.0], 4950, 7, 45000]"),
+            "page size {ps}"
+        );
+        h5dump_ok(&path);
+    }
+}
+
+// ---- 4. fill time and fill value ----
+
+fn fill_message(bytes: &[u8], path: &str) -> clawhdf5_format::object_header::HeaderMessage {
+    let (_, oh) = header_at(bytes, path);
+    oh.messages
+        .into_iter()
+        .find(|m| m.msg_type == MessageType::FillValue)
+        .unwrap()
+}
+
+fn fill_file() -> Vec<u8> {
+    let mut fw = FileWriter::new();
+    fw.create_dataset("never")
+        .with_f64_data(&[1.0, 2.0])
+        .fill_time(FillTime::Never);
+    fw.create_dataset("alloc")
+        .with_f64_data(&[1.0, 2.0])
+        .fill_time(FillTime::Alloc);
+    fw.create_dataset("ifset")
+        .with_f64_data(&[1.0, 2.0])
+        .fill_time(FillTime::IfSet);
+    fw.create_dataset("default").with_f64_data(&[1.0, 2.0]);
+    fw.create_dataset("filled")
+        .with_i32_data(&[1, 2, 3, 4])
+        .with_chunks(&[2])
+        .with_maxshape(&[u64::MAX])
+        .with_fill_value(&(-1i32).to_le_bytes());
+    fw.finish().unwrap()
+}
+
+#[test]
+fn fill_time_uses_libhdf5_codes() {
+    // H5D_FILL_TIME_ALLOC = 0, NEVER = 1, IFSET = 2, in bits 2-3. Measured:
+    // h5py saw our Never as ALLOC, Alloc as IFSET and IfSet as NEVER.
+    let bytes = fill_file();
+    for (path, code) in [("never", 1), ("alloc", 0), ("ifset", 2), ("default", 2)] {
+        let msg = fill_message(&bytes, path);
+        assert_eq!((msg.data[1] >> 2) & 3, code, "{path}");
+        assert_eq!(msg.data[1] & 3, 2, "{path}: allocation time stays late");
+    }
+    for ft in [FillTime::Never, FillTime::Alloc, FillTime::IfSet] {
+        assert_eq!(FillTime::from_byte(ft.to_byte()), Some(ft));
+    }
+    assert_eq!(FillTime::default(), FillTime::IfSet);
+}
+
+#[test]
+fn fill_value_is_written_and_read_back() {
+    let bytes = fill_file();
+    let msg = fill_message(&bytes, "filled");
+    assert_eq!(
+        clawhdf5_format::fill_value::parse_fill_value(&msg).unwrap(),
+        Some((-1i32).to_le_bytes().to_vec())
+    );
+    assert_eq!(
+        clawhdf5_format::fill_value::parse_fill_value(&fill_message(&bytes, "ifset")).unwrap(),
+        None
+    );
+
+    // One element's bytes, no more, no less.
+    let mut fw = FileWriter::new();
+    fw.create_dataset("d")
+        .with_f64_data(&[1.0])
+        .with_fill_value(&[0; 4]);
+    assert!(fw.finish().is_err());
+}
+
+#[test]
+#[ignore = "requires Python h5py module and h5dump"]
+fn h5py_sees_our_fill_time_and_fill_value() {
+    let path = write_tmp("fill", &fill_file());
+    let out = h5py(
+        &path,
+        "from h5py import h5d\n\
+         f = h5py.File(path, 'r')\n\
+         names = {h5d.FILL_TIME_NEVER: 'never', h5d.FILL_TIME_ALLOC: 'alloc', h5d.FILL_TIME_IFSET: 'ifset'}\n\
+         t = [names[f[n].id.get_create_plist().get_fill_time()] for n in ('never', 'alloc', 'ifset', 'default')]\n\
+         print(json.dumps([t, int(f['filled'].fillvalue), f['filled'][()].tolist()]))\n\
+         f.close()\n\
+         f = h5py.File(path, 'r+')\n\
+         f['filled'].resize((7,))\n\
+         f.close()\n\
+         print(json.dumps(h5py.File(path, 'r')['filled'][()].tolist()))",
+    );
+    assert_eq!(
+        out,
+        "[[\"never\", \"alloc\", \"ifset\", \"ifset\"], -1, [1, 2, 3, 4]]\n[1, 2, 3, 4, -1, -1, -1]"
+    );
+    h5dump_ok(&path);
+}
+
+// ---- 5. empty string attributes ----
+
+fn empty_string_file() -> Vec<u8> {
+    let mut fw = FileWriter::new();
+    fw.set_root_attr("empty", AttrValue::String(String::new()));
+    fw.set_root_attr("x", AttrValue::String("héllo".into()));
+    fw.set_root_attr(
+        "empties",
+        AttrValue::StringArray(vec![String::new(), String::new()]),
+    );
+    fw.set_root_attr("n", AttrValue::I64(3));
+    fw.finish().unwrap()
+}
+
+#[test]
+fn empty_string_attribute_has_a_one_byte_type() {
+    // Measured: "" got a size-0 string type, and libhdf5 then refused every
+    // attribute on the object ("invalid datatype size").
+    let bytes = empty_string_file();
+    let (sb, root) = header_at(&bytes, "/");
+    let attrs = clawhdf5_format::attribute::extract_attributes(&root, sb.length_size).unwrap();
+    for name in ["empty", "empties"] {
+        let a = attrs.iter().find(|a| a.name == name).unwrap();
+        assert_eq!(a.datatype.type_size(), 1, "{name}");
+        let strings = a.read_as_strings().unwrap();
+        assert!(strings.iter().all(String::is_empty), "{name}: {strings:?}");
+    }
+
+    // A size-0 string type handed in directly is refused, not written.
+    let mut fw = FileWriter::new();
+    fw.set_root_attr(
+        "raw",
+        AttrValue::Raw {
+            datatype: Datatype::String {
+                size: 0,
+                padding: clawhdf5_format::datatype::StringPadding::NullPad,
+                charset: clawhdf5_format::datatype::CharacterSet::Ascii,
+            },
+            shape: vec![],
+            data: vec![],
+        },
+    );
+    assert!(fw.finish().is_err());
+}
+
+#[test]
+#[ignore = "requires Python h5py module and h5dump"]
+fn h5py_reads_all_attributes_next_to_an_empty_string() {
+    let path = write_tmp("empty_str", &empty_string_file());
+    let out = h5py(
+        &path,
+        "f = h5py.File(path, 'r')\n\
+         d = lambda v: v.decode() if isinstance(v, bytes) else v\n\
+         print(json.dumps([d(f.attrs['empty']), d(f.attrs['x']),\n\
+         \x20   [d(s) for s in f.attrs['empties']], int(f.attrs['n'])], ensure_ascii=False))",
+    );
+    assert_eq!(out, r#"["", "héllo", ["", ""], 3]"#);
+    h5dump_ok(&path);
+}
+
+// ---- 6. path-like names ----
+
+#[test]
+fn slash_in_a_group_or_dataset_name_is_an_error() {
+    // Measured: create_group("a/b") wrote one link literally named "a/b",
+    // which h5py cannot reach ("component not found"). The writer has no
+    // nested groups, so such names are refused.
+    let mut fw = FileWriter::new();
+    let mut g = fw.create_group("a/b");
+    g.create_dataset("c").with_f64_data(&[1.0]);
+    fw.add_group(g.finish());
+    assert!(fw.finish().is_err());
+
+    let mut fw = FileWriter::new();
+    fw.create_dataset("x/y").with_f64_data(&[1.0]);
+    assert!(fw.finish().is_err());
+
+    let mut fw = FileWriter::new();
+    let mut g = fw.create_group("g");
+    g.create_dataset("x/y").with_f64_data(&[1.0]);
+    fw.add_group(g.finish());
+    assert!(fw.finish().is_err());
+
+    for bad in ["", "."] {
+        let mut fw = FileWriter::new();
+        fw.create_dataset(bad).with_f64_data(&[1.0]);
+        assert!(fw.finish().is_err(), "{bad:?}");
+    }
+
+    // One level of groups still works, and '/' stays legal in attribute names.
+    let mut fw = FileWriter::new();
+    let mut g = fw.create_group("g");
+    g.create_dataset("c").with_f64_data(&[1.0]);
+    g.set_attr("m/s", AttrValue::I64(1));
+    fw.add_group(g.finish());
+    let bytes = fw.finish().unwrap();
+    header_at(&bytes, "g/c");
+}
+
+// ---- 7. unknown-message flags on read ----
+
+#[test]
+fn unknown_message_flags_follow_libhdf5_on_tbogus() {
+    // libhdf5's own test file (test/testfiles/tbogus.h5): datasets carrying
+    // an unknown message with various flags. libhdf5 (read-only) opens
+    // Dataset1, 2, 4 and 5 and refuses Dataset3 ("unknown message with 'fail
+    // if unknown' flag found"). We used to refuse Dataset2 (bit 3, which only
+    // applies when writing) and open Dataset3 (bit 7, fail always).
+    let bytes = include_bytes!("fixtures/tbogus.h5");
+    let sig = signature::find_signature(bytes).unwrap();
+    let sb = Superblock::parse(bytes, sig).unwrap();
+    for (name, readable) in [
+        ("Dataset1", true),
+        ("Dataset2", true),
+        ("Dataset3", false),
+        ("Dataset4", true),
+        ("Dataset5", true),
+    ] {
+        let addr = resolve_path_any(bytes, &sb, name).unwrap();
+        let parsed = ObjectHeader::parse(bytes, addr as usize, sb.offset_size, sb.length_size);
+        match parsed {
+            Ok(_) => assert!(readable, "{name} must be refused"),
+            Err(e) => {
+                assert!(!readable, "{name} must be readable, got {e:?}");
+                assert!(matches!(
+                    e,
+                    clawhdf5_format::error::FormatError::UnsupportedMessage(_)
+                ));
+            }
+        }
+    }
+}
+
+// ---- 8. shared fill value messages ----
+
+#[test]
+fn shared_fill_value_is_resolved_not_zero() {
+    // gen_shared_fill.py: HDF5 2.0 with a SOHM index for fill values, so each
+    // dataset's fill value message is a reference into the SOHM heap. It
+    // used to be read as "no fill value" (zeros) instead of -7.
+    let bytes = include_bytes!("fixtures/shared_fill_value.h5");
+    for (name, shared) in [
+        ("sohm_a", false),
+        ("sohm_b", true),
+        ("unwritten_a", false),
+        ("unwritten_b", true),
+    ] {
+        let (sb, oh) = header_at(bytes, name);
+        let msg = oh
+            .messages
+            .iter()
+            .find(|m| m.msg_type == MessageType::FillValue)
+            .unwrap();
+        assert_eq!(
+            clawhdf5_format::shared_message::is_shared(msg.flags),
+            shared,
+            "{name}: fixture layout"
+        );
+        if shared {
+            // Without the file the reference cannot be followed: an error,
+            // never a silent default.
+            assert_eq!(
+                clawhdf5_format::fill_value::dataset_fill_value(&oh.messages),
+                Err(clawhdf5_format::error::FormatError::UnresolvedSharedMessage)
+            );
+        }
+        assert_eq!(
+            clawhdf5_format::fill_value::dataset_fill_value_in(
+                bytes,
+                &oh.messages,
+                sb.offset_size,
+                sb.length_size
+            )
+            .unwrap(),
+            Some((-7i32).to_le_bytes().to_vec()),
+            "{name}"
+        );
+    }
+}

crates/clawhdf5-gpu/Cargo.toml

@@ -2,6 +2,7 @@
 name = "clawhdf5-gpu"
 version = "2.7.0"
 edition = "2024"
+rust-version.workspace = true
 description = "GPU-accelerated vector operations for rustyhdf5 using wgpu compute shaders"
 license = "MIT"
 repository = "https://git.redclaw.dev/quantumclaw/clawhdf5"

crates/clawhdf5-io/Cargo.toml

@@ -2,6 +2,7 @@
 name = "clawhdf5-io"
 version = "2.7.0"
 edition = "2024"
+rust-version.workspace = true
 description = "I/O abstraction layer for rustyhdf5"
 license = "MIT"
 repository = "https://git.redclaw.dev/quantumclaw/clawhdf5"

crates/clawhdf5-migrate/Cargo.toml

@@ -2,7 +2,8 @@
 name = "clawhdf5-migrate"
 version = "2.7.0"
 edition = "2024"
-description = "CLI to migrate SQLite agent memory databases to HDF5 format"
+rust-version.workspace = true
+description = "CLI to migrate SQLite agent memory databases to clawhdf5-agent stores"
 license = "MIT"
 repository = "https://git.redclaw.dev/quantumclaw/clawhdf5"
 readme = "README.md"
@@ -16,10 +17,8 @@ path = "src/main.rs"
 [dependencies]
 clawhdf5-agent = { path = "../clawhdf5-agent", version = "2.7.0" }
 clawhdf5-format = { path = "../clawhdf5-format", version = "2.7.0" }
-clawhdf5 = { path = "../clawhdf5", version = "2.7.0" }
 rusqlite = { version = "0.31", features = ["bundled"] }
 clap = { version = "4", features = ["derive"] }
-half = { workspace = true }
 
 [dev-dependencies]
 tempfile = { workspace = true }

crates/clawhdf5-migrate/README.md

@@ -3,9 +3,15 @@
 [![crates.io](https://img.shields.io/crates/v/clawhdf5-migrate.svg)](https://crates.io/crates/clawhdf5-migrate)
 [![docs.rs](https://img.shields.io/docsrs/clawhdf5-migrate)](https://docs.rs/clawhdf5-migrate)
 
-CLI tool to migrate SQLite agent memory databases to HDF5 format.
+CLI tool to migrate a SQLite agent-memory database in the `memory_chunks` / `sessions` / `entities` / `relations` layout (table and
+column names are configurable) to a
+[clawhdf5-agent](https://crates.io/crates/clawhdf5-agent) store. This is **not**
+ZeroClaw's schema — ZeroClaw keeps memories in a single `memories` table and
+does not use clawhdf5.
 
-Converts existing SQLite-based agent memory stores (embeddings, text chunks, metadata) into the HDF5 format used by [clawhdf5-agent](https://crates.io/crates/clawhdf5-agent).
+The output is written through `clawhdf5-agent`'s own API, so it opens with
+`HDF5Memory::open` and is searchable immediately: memory records, sessions and
+the knowledge graph (entities and relations) are carried over.
 
 ## Installation
 
@@ -16,9 +22,19 @@ cargo install clawhdf5-migrate
 ## Usage
 
 ```bash
-clawhdf5-migrate --input agent.db --output agent.h5
+clawhdf5-migrate --sqlite agent.db --hdf5 agent.h5 --agent-id my-agent
 ```
 
+Embeddings are stored as float16 (the library default for new stores); pass
+`--f32` for full precision. Every embedding must have the same dimension
+(the first row's, or `--embedding-dim`, which a source with no memory records
+requires); rows are never truncated, and the whole source is checked before an
+existing output store is replaced. `--incremental` adds only new rows to an
+existing store of the same dimension and carries over changes to rows'
+deleted flags, `--skip-deleted` leaves out tombstoned rows, and `--dry-run`
+only counts.
+See `clawhdf5-migrate --help` for every option.
+
 ## License
 
 MIT

crates/clawhdf5-migrate/src/hdf5_reader.rs

@@ -1,163 +0,0 @@
-//! Read a migration HDF5 file back into the in-memory data model.
-//!
-//! Used to verify migrated content (real validation) and to merge new rows into
-//! an existing output (incremental migration). Mirrors the layout produced by
-//! [`crate::hdf5_writer`].
-
-use clawhdf5::reader::{File, Group};
-use clawhdf5_format::type_builders::AttrValue;
-
-use crate::sqlite_reader::{Entity, MemoryChunk, Relation, Session, SqliteData};
-
-type BoxErr = Box<dyn std::error::Error>;
-
-fn read_strings(group: &Group<'_>, name: &str) -> Result<Vec<String>, BoxErr> {
-    Ok(group.dataset(name)?.read_string()?)
-}
-
-fn read_i64s(group: &Group<'_>, name: &str) -> Result<Vec<i64>, BoxErr> {
-    Ok(group.dataset(name)?.read_i64()?)
-}
-
-fn read_f64s(group: &Group<'_>, name: &str) -> Result<Vec<f64>, BoxErr> {
-    Ok(group.dataset(name)?.read_f64()?)
-}
-
-/// Read the embeddings dataset as a flat `Vec<f32>` of `n * dim` values,
-/// handling both f32 and (lossy) f16 storage.
-fn read_embeddings_flat(group: &Group<'_>) -> Result<Vec<f32>, BoxErr> {
-    Ok(group.dataset("embeddings")?.read_f32()?)
-}
-
-/// Read a migration HDF5 file into a [`SqliteData`].
-pub fn read_hdf5(path: &str) -> Result<SqliteData, BoxErr> {
-    let file = File::open(path)?;
-
-    let embedding_dim = match file.root().attrs()?.get("embedding_dim") {
-        Some(AttrValue::I64(d)) => *d as usize,
-        _ => 0,
-    };
-
-    let chunks = read_chunks(&file, embedding_dim)?;
-    let sessions = read_sessions(&file)?;
-    let entities = read_entities(&file)?;
-    let relations = read_relations(&file)?;
-
-    Ok(SqliteData {
-        chunks,
-        sessions,
-        entities,
-        relations,
-        embedding_dim,
-        // Not a SQLite read — the caller (incremental migration) carries
-        // forward the current run's actual `source_path` from the fresh
-        // SQLite read instead of using this placeholder.
-        source_path: String::new(),
-    })
-}
-
-fn read_chunks(file: &File, dim: usize) -> Result<Vec<MemoryChunk>, BoxErr> {
-    let g = file.group("chunks")?;
-    let count = group_count(&g)?;
-    if count == 0 {
-        return Ok(Vec::new());
-    }
-    let ids = read_i64s(&g, "id")?;
-    let texts = read_strings(&g, "text")?;
-    let channels = read_strings(&g, "source_channel")?;
-    let timestamps = read_f64s(&g, "timestamp")?;
-    let session_ids = read_strings(&g, "session_id")?;
-    let tags = read_strings(&g, "tags")?;
-    let deleted = g.dataset("deleted")?.read_i32()?;
-    let emb_flat = read_embeddings_flat(&g)?;
-    let dim = dim.max(1);
-
-    let mut chunks = Vec::with_capacity(ids.len());
-    for (i, &id) in ids.iter().enumerate() {
-        let embedding = emb_flat
-            .get(i * dim..(i + 1) * dim)
-            .map(|s| s.to_vec())
-            .unwrap_or_default();
-        chunks.push(MemoryChunk {
-            id,
-            chunk: texts.get(i).cloned().unwrap_or_default(),
-            embedding,
-            source_channel: channels.get(i).cloned().unwrap_or_default(),
-            timestamp: timestamps.get(i).copied().unwrap_or(0.0),
-            session_id: session_ids.get(i).cloned().unwrap_or_default(),
-            tags: tags.get(i).cloned().unwrap_or_default(),
-            deleted: deleted.get(i).copied().unwrap_or(0),
-        });
-    }
-    Ok(chunks)
-}
-
-fn read_sessions(file: &File) -> Result<Vec<Session>, BoxErr> {
-    let g = file.group("sessions")?;
-    if group_count(&g)? == 0 {
-        return Ok(Vec::new());
-    }
-    let ids = read_strings(&g, "id")?;
-    let starts = read_i64s(&g, "start_idx")?;
-    let ends = read_i64s(&g, "end_idx")?;
-    let channels = read_strings(&g, "channel")?;
-    let timestamps = read_f64s(&g, "timestamp")?;
-    let summaries = read_strings(&g, "summary")?;
-    Ok((0..ids.len())
-        .map(|i| Session {
-            id: ids[i].clone(),
-            start_idx: starts.get(i).copied().unwrap_or(0),
-            end_idx: ends.get(i).copied().unwrap_or(0),
-            channel: channels.get(i).cloned().unwrap_or_default(),
-            timestamp: timestamps.get(i).copied().unwrap_or(0.0),
-            summary: summaries.get(i).cloned().unwrap_or_default(),
-        })
-        .collect())
-}
-
-fn read_entities(file: &File) -> Result<Vec<Entity>, BoxErr> {
-    let g = file.group("entities")?;
-    if group_count(&g)? == 0 {
-        return Ok(Vec::new());
-    }
-    let ids = read_i64s(&g, "id")?;
-    let names = read_strings(&g, "name")?;
-    let types = read_strings(&g, "type")?;
-    let emb_idxs = read_i64s(&g, "embedding_idx")?;
-    Ok((0..ids.len())
-        .map(|i| Entity {
-            id: ids[i],
-            name: names.get(i).cloned().unwrap_or_default(),
-            entity_type: types.get(i).cloned().unwrap_or_default(),
-            embedding_idx: emb_idxs.get(i).copied().unwrap_or(-1),
-        })
-        .collect())
-}
-
-fn read_relations(file: &File) -> Result<Vec<Relation>, BoxErr> {
-    let g = file.group("relations")?;
-    if group_count(&g)? == 0 {
-        return Ok(Vec::new());
-    }
-    let srcs = read_i64s(&g, "src")?;
-    let tgts = read_i64s(&g, "tgt")?;
-    let rels = read_strings(&g, "relation")?;
-    let weights = read_f64s(&g, "weight")?;
-    let timestamps = read_f64s(&g, "timestamp")?;
-    Ok((0..srcs.len())
-        .map(|i| Relation {
-            src: srcs[i],
-            tgt: tgts.get(i).copied().unwrap_or(0),
-            relation: rels.get(i).cloned().unwrap_or_default(),
-            weight: weights.get(i).copied().unwrap_or(1.0),
-            timestamp: timestamps.get(i).copied().unwrap_or(0.0),
-        })
-        .collect())
-}
-
-fn group_count(group: &Group<'_>) -> Result<u64, BoxErr> {
-    match group.attrs()?.get("count") {
-        Some(AttrValue::I64(n)) => Ok(*n as u64),
-        _ => Ok(0),
-    }
-}

crates/clawhdf5-migrate/src/hdf5_writer.rs

@@ -1,366 +0,0 @@
-use clawhdf5::writer::FileBuilder;
-use clawhdf5_format::datatype::{CharacterSet, Datatype, StringPadding};
-use clawhdf5_format::type_builders::AttrValue;
-
-use crate::sqlite_reader::SqliteData;
-
-/// Options controlling HDF5 output.
-pub struct WriteOptions {
-    pub agent_id: String,
-    pub embedder: String,
-    pub compression: bool,
-    pub compression_level: u32,
-    pub float16: bool,
-}
-
-/// Write SQLite data to an HDF5 file.
-pub fn write_hdf5(
-    path: &str,
-    data: &SqliteData,
-    opts: &WriteOptions,
-) -> Result<(), Box<dyn std::error::Error>> {
-    let mut builder = FileBuilder::new();
-    let timestamp = iso8601_now();
-
-    // Root-level metadata attributes
-    builder.set_attr("agent_id", AttrValue::String(opts.agent_id.clone()));
-    builder.set_attr("embedder", AttrValue::String(opts.embedder.clone()));
-    builder.set_attr("embedding_dim", AttrValue::I64(data.embedding_dim as i64));
-    builder.set_attr("source", AttrValue::String("sqlite-migration".into()));
-    builder.set_attr("version", AttrValue::I64(1));
-    // Lineage: which SQLite database this output was migrated from and when,
-    // plus the migrator tool version — so a chain of `--incremental` runs
-    // still has an audit trail instead of every run overwriting the same
-    // static attributes (see research/03_provenance.md, INT-03).
-    builder.set_attr("source_path", AttrValue::String(data.source_path.clone()));
-    builder.set_attr("migrated_at", AttrValue::String(timestamp.clone()));
-    builder.set_attr(
-        "migrator_version",
-        AttrValue::String(env!("CARGO_PKG_VERSION").to_owned()),
-    );
-
-    write_chunks_group(&mut builder, data, opts, &timestamp);
-    write_sessions_group(&mut builder, data);
-    write_entities_group(&mut builder, data);
-    write_relations_group(&mut builder, data);
-
-    builder.write(path)?;
-    Ok(())
-}
-
-/// Current UTC time formatted as an ISO-8601 / RFC-3339 timestamp
-/// (`YYYY-MM-DDTHH:MM:SSZ`), with no external date/time dependency.
-fn iso8601_now() -> String {
-    let secs = std::time::SystemTime::now()
-        .duration_since(std::time::UNIX_EPOCH)
-        .unwrap_or_default()
-        .as_secs();
-    let days = (secs / 86_400) as i64;
-    let time_of_day = secs % 86_400;
-    let (h, m, s) = (
-        time_of_day / 3600,
-        (time_of_day % 3600) / 60,
-        time_of_day % 60,
-    );
-    let (y, mo, d) = civil_from_days(days);
-    format!("{y:04}-{mo:02}-{d:02}T{h:02}:{m:02}:{s:02}Z")
-}
-
-/// Days-since-epoch to (year, month, day), Howard Hinnant's `civil_from_days`
-/// algorithm (proleptic Gregorian calendar, valid for the full `i64` range).
-fn civil_from_days(z: i64) -> (i64, u32, u32) {
-    let z = z + 719_468;
-    let era = if z >= 0 { z } else { z - 146_096 } / 146_097;
-    let doe = (z - era * 146_097) as u64; // [0, 146096]
-    let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365; // [0, 399]
-    let y = yoe as i64 + era * 400;
-    let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); // [0, 365]
-    let mp = (5 * doy + 2) / 153; // [0, 11]
-    let d = (doy - (153 * mp + 2) / 5 + 1) as u32; // [1, 31]
-    let m = (if mp < 10 { mp + 3 } else { mp - 9 }) as u32; // [1, 12]
-    let y = if m <= 2 { y + 1 } else { y };
-    (y, m, d)
-}
-
-/// Build a fixed-length string Datatype from the max byte length of the items.
-fn string_dtype(max_len: usize) -> Datatype {
-    Datatype::String {
-        size: max_len.max(1) as u32,
-        padding: StringPadding::NullPad,
-        charset: CharacterSet::Utf8,
-    }
-}
-
-/// Pack a slice of strings into null-padded raw bytes of uniform width.
-fn pack_strings(strings: &[String]) -> (Vec<u8>, usize) {
-    let max_len = strings.iter().map(|s| s.len()).max().unwrap_or(0).max(1);
-    let mut buf = vec![0u8; strings.len() * max_len];
-    for (i, s) in strings.iter().enumerate() {
-        let start = i * max_len;
-        let bytes = s.as_bytes();
-        let copy_len = bytes.len().min(max_len);
-        buf[start..start + copy_len].copy_from_slice(&bytes[..copy_len]);
-    }
-    (buf, max_len)
-}
-
-fn apply_compression(ds: &mut clawhdf5_format::type_builders::DatasetBuilder, opts: &WriteOptions) {
-    if opts.compression {
-        ds.with_deflate(opts.compression_level);
-        ds.with_shuffle();
-    }
-}
-
-fn write_chunks_group(
-    builder: &mut FileBuilder,
-    data: &SqliteData,
-    opts: &WriteOptions,
-    timestamp: &str,
-) {
-    let mut group = builder.create_group("chunks");
-    let n = data.chunks.len() as u64;
-
-    if n == 0 {
-        group.set_attr("count", AttrValue::I64(0));
-        builder.add_group(group.finish());
-        return;
-    }
-
-    group.set_attr("count", AttrValue::I64(n as i64));
-
-    // Source attribution attached directly to the content-bearing datasets
-    // (SHA-256 of the raw bytes + creator/timestamp/source), so the chunk
-    // text and embeddings each carry their own verifiable provenance
-    // (see clawhdf5_format::provenance / `Dataset::verify_provenance`).
-    let source_opt = if data.source_path.is_empty() {
-        None
-    } else {
-        Some(data.source_path.as_str())
-    };
-
-    // ids
-    let ids: Vec<i64> = data.chunks.iter().map(|c| c.id).collect();
-    group.create_dataset("id").with_i64_data(&ids);
-
-    // text
-    let texts: Vec<String> = data.chunks.iter().map(|c| c.chunk.clone()).collect();
-    let (text_raw, text_len) = pack_strings(&texts);
-    group
-        .create_dataset("text")
-        .with_compound_data(string_dtype(text_len), text_raw, n)
-        .with_provenance("clawhdf5-migrate", timestamp, source_opt);
-
-    // embeddings - flatten to [N, dim]
-    let dim = data.embedding_dim;
-    if opts.float16 {
-        let f16_data: Vec<u16> = data
-            .chunks
-            .iter()
-            .flat_map(|c| {
-                c.embedding
-                    .iter()
-                    .map(|&v| half::f16::from_f32(v).to_bits())
-            })
-            .collect();
-        let raw: Vec<u8> = f16_data.iter().flat_map(|v| v.to_le_bytes()).collect();
-        let f16_dtype = Datatype::FloatingPoint {
-            size: 2,
-            byte_order: clawhdf5_format::datatype::DatatypeByteOrder::LittleEndian,
-            bit_offset: 0,
-            bit_precision: 16,
-            exponent_location: 10,
-            exponent_size: 5,
-            mantissa_location: 0,
-            mantissa_size: 10,
-            exponent_bias: 15,
-        };
-        let ds = group
-            .create_dataset("embeddings")
-            .with_compound_data(f16_dtype, raw, n)
-            .with_shape(&[n, dim as u64])
-            .with_provenance("clawhdf5-migrate", timestamp, source_opt);
-        apply_compression(ds, opts);
-    } else {
-        let flat: Vec<f32> = data
-            .chunks
-            .iter()
-            .flat_map(|c| c.embedding.iter().copied())
-            .collect();
-        let ds = group
-            .create_dataset("embeddings")
-            .with_f32_data(&flat)
-            .with_shape(&[n, dim as u64])
-            .with_provenance("clawhdf5-migrate", timestamp, source_opt);
-        apply_compression(ds, opts);
-    }
-
-    // source_channel
-    let channels: Vec<String> = data
-        .chunks
-        .iter()
-        .map(|c| c.source_channel.clone())
-        .collect();
-    let (ch_raw, ch_len) = pack_strings(&channels);
-    group
-        .create_dataset("source_channel")
-        .with_compound_data(string_dtype(ch_len), ch_raw, n);
-
-    // timestamp
-    let timestamps: Vec<f64> = data.chunks.iter().map(|c| c.timestamp).collect();
-    group.create_dataset("timestamp").with_f64_data(&timestamps);
-
-    // session_id
-    let sess_ids: Vec<String> = data.chunks.iter().map(|c| c.session_id.clone()).collect();
-    let (sid_raw, sid_len) = pack_strings(&sess_ids);
-    group
-        .create_dataset("session_id")
-        .with_compound_data(string_dtype(sid_len), sid_raw, n);
-
-    // tags
-    let tags: Vec<String> = data.chunks.iter().map(|c| c.tags.clone()).collect();
-    let (tag_raw, tag_len) = pack_strings(&tags);
-    group
-        .create_dataset("tags")
-        .with_compound_data(string_dtype(tag_len), tag_raw, n);
-
-    // deleted
-    let deleted: Vec<i32> = data.chunks.iter().map(|c| c.deleted).collect();
-    group.create_dataset("deleted").with_i32_data(&deleted);
-
-    builder.add_group(group.finish());
-}
-
-fn write_sessions_group(builder: &mut FileBuilder, data: &SqliteData) {
-    let mut group = builder.create_group("sessions");
-    let n = data.sessions.len() as u64;
-    group.set_attr("count", AttrValue::I64(n as i64));
-
-    if n == 0 {
-        builder.add_group(group.finish());
-        return;
-    }
-
-    let ids: Vec<String> = data.sessions.iter().map(|s| s.id.clone()).collect();
-    let (id_raw, id_len) = pack_strings(&ids);
-    group
-        .create_dataset("id")
-        .with_compound_data(string_dtype(id_len), id_raw, n);
-
-    let start_idxs: Vec<i64> = data.sessions.iter().map(|s| s.start_idx).collect();
-    group.create_dataset("start_idx").with_i64_data(&start_idxs);
-
-    let end_idxs: Vec<i64> = data.sessions.iter().map(|s| s.end_idx).collect();
-    group.create_dataset("end_idx").with_i64_data(&end_idxs);
-
-    let channels: Vec<String> = data.sessions.iter().map(|s| s.channel.clone()).collect();
-    let (ch_raw, ch_len) = pack_strings(&channels);
-    group
-        .create_dataset("channel")
-        .with_compound_data(string_dtype(ch_len), ch_raw, n);
-
-    let timestamps: Vec<f64> = data.sessions.iter().map(|s| s.timestamp).collect();
-    group.create_dataset("timestamp").with_f64_data(&timestamps);
-
-    let summaries: Vec<String> = data.sessions.iter().map(|s| s.summary.clone()).collect();
-    let (sum_raw, sum_len) = pack_strings(&summaries);
-    group
-        .create_dataset("summary")
-        .with_compound_data(string_dtype(sum_len), sum_raw, n);
-
-    builder.add_group(group.finish());
-}
-
-fn write_entities_group(builder: &mut FileBuilder, data: &SqliteData) {
-    let mut group = builder.create_group("entities");
-    let n = data.entities.len() as u64;
-    group.set_attr("count", AttrValue::I64(n as i64));
-
-    if n == 0 {
-        builder.add_group(group.finish());
-        return;
-    }
-
-    let ids: Vec<i64> = data.entities.iter().map(|e| e.id).collect();
-    group.create_dataset("id").with_i64_data(&ids);
-
-    let names: Vec<String> = data.entities.iter().map(|e| e.name.clone()).collect();
-    let (name_raw, name_len) = pack_strings(&names);
-    group
-        .create_dataset("name")
-        .with_compound_data(string_dtype(name_len), name_raw, n);
-
-    let types: Vec<String> = data
-        .entities
-        .iter()
-        .map(|e| e.entity_type.clone())
-        .collect();
-    let (type_raw, type_len) = pack_strings(&types);
-    group
-        .create_dataset("type")
-        .with_compound_data(string_dtype(type_len), type_raw, n);
-
-    let emb_idxs: Vec<i64> = data.entities.iter().map(|e| e.embedding_idx).collect();
-    group
-        .create_dataset("embedding_idx")
-        .with_i64_data(&emb_idxs);
-
-    builder.add_group(group.finish());
-}
-
-fn write_relations_group(builder: &mut FileBuilder, data: &SqliteData) {
-    let mut group = builder.create_group("relations");
-    let n = data.relations.len() as u64;
-    group.set_attr("count", AttrValue::I64(n as i64));
-
-    if n == 0 {
-        builder.add_group(group.finish());
-        return;
-    }
-
-    let srcs: Vec<i64> = data.relations.iter().map(|r| r.src).collect();
-    group.create_dataset("src").with_i64_data(&srcs);
-
-    let tgts: Vec<i64> = data.relations.iter().map(|r| r.tgt).collect();
-    group.create_dataset("tgt").with_i64_data(&tgts);
-
-    let rels: Vec<String> = data.relations.iter().map(|r| r.relation.clone()).collect();
-    let (rel_raw, rel_len) = pack_strings(&rels);
-    group
-        .create_dataset("relation")
-        .with_compound_data(string_dtype(rel_len), rel_raw, n);
-
-    let weights: Vec<f64> = data.relations.iter().map(|r| r.weight).collect();
-    group.create_dataset("weight").with_f64_data(&weights);
-
-    let timestamps: Vec<f64> = data.relations.iter().map(|r| r.timestamp).collect();
-    group.create_dataset("timestamp").with_f64_data(&timestamps);
-
-    builder.add_group(group.finish());
-}
-
-#[cfg(test)]
-mod time_tests {
-    use super::civil_from_days;
-
-    #[test]
-    fn epoch_day_zero_is_1970_01_01() {
-        assert_eq!(civil_from_days(0), (1970, 1, 1));
-    }
-
-    #[test]
-    fn known_dates_roundtrip() {
-        // 2026-08-16 is 20,681 days after 1970-01-01.
-        assert_eq!(civil_from_days(20_681), (2026, 8, 16));
-        // 2000-02-29 (leap day itself) and 2000-03-01 (the day after).
-        assert_eq!(civil_from_days(11_016), (2000, 2, 29));
-        assert_eq!(civil_from_days(11_017), (2000, 3, 1));
-    }
-
-    #[test]
-    fn iso8601_now_has_expected_shape() {
-        let ts = super::iso8601_now();
-        assert_eq!(ts.len(), "2026-08-16T00:00:00Z".len());
-        assert!(ts.starts_with("20")); // sanity: 21st-century year
-        assert!(ts.ends_with('Z'));
-    }
-}

crates/clawhdf5-migrate/src/sqlite_reader.rs

@@ -43,19 +43,15 @@ pub struct Relation {
     pub timestamp: f64,
 }
 
-/// All data read from a ZeroClaw SQLite database.
+/// All data read from a source SQLite database.
 #[derive(Debug)]
 pub struct SqliteData {
     pub chunks: Vec<MemoryChunk>,
     pub sessions: Vec<Session>,
     pub entities: Vec<Entity>,
     pub relations: Vec<Relation>,
+    /// `--embedding-dim`, or the first row's; 0 when neither exists.
     pub embedding_dim: usize,
-    /// Filesystem path of the SQLite database this data was read from, for
-    /// provenance attribution on the HDF5 output. Empty when the data did
-    /// not come directly from a SQLite read (e.g. re-read of a prior HDF5
-    /// migration output for an incremental merge).
-    pub source_path: String,
 }
 
 /// A table name plus the ordered column names the reader maps by position.
@@ -67,7 +63,9 @@ pub struct TableSchema {
 
 /// Configurable mapping from a SQLite layout to the migration's data model.
 ///
-/// Defaults to the ZeroClaw schema; the CLI can override the table names so the
+/// Defaults to the `memory_chunks` / `sessions` / `entities` / `relations`
+/// layout (not ZeroClaw's schema, despite what earlier docs said); the CLI can
+/// override the table names so the
 /// tool can migrate databases whose tables are named differently. Column names
 /// (and order) are part of the config too, so a library caller can remap them.
 #[derive(Debug, Clone)]
@@ -167,11 +165,13 @@ pub fn read_counts(
     })
 }
 
-/// Auto-detect embedding dimension from the first chunk's BLOB size.
+/// Auto-detect embedding dimension from the BLOB size of the first chunk (in
+/// id order, deleted or not).
 fn detect_embedding_dim(conn: &Connection, config: &SchemaConfig) -> SqlResult<Option<usize>> {
     let emb_col = config.chunks.columns.get(2).copied().unwrap_or("embedding");
+    let id_col = config.chunks.columns.first().copied().unwrap_or("id");
     let mut stmt = conn.prepare(&format!(
-        "SELECT {emb_col} FROM {} LIMIT 1",
+        "SELECT {emb_col} FROM {} ORDER BY {id_col} LIMIT 1",
         config.chunks.table
     ))?;
     let mut rows = stmt.query([])?;
@@ -192,27 +192,19 @@ fn blob_to_f32(blob: &[u8]) -> Vec<f32> {
         .collect()
 }
 
-/// Read all data from a ZeroClaw SQLite database.
+/// Read all data from a source SQLite database.
 ///
 /// If `skip_deleted` is true, rows with `deleted=1` are excluded from chunks.
-/// If `embedding_dim` is `None`, auto-detect from the first row.
+/// If `embedding_dim` is `None`, auto-detect from the first row (0 when there
+/// are no rows). Embeddings are returned at their full stored length whatever
+/// the dimension: checking that every row matches it is the writer's job
+/// (`store_writer::write_store`), so a mismatch is an error, not silent
+/// truncation.
 pub fn read_sqlite(
     path: &str,
     skip_deleted: bool,
     embedding_dim: Option<usize>,
     config: &SchemaConfig,
-) -> Result<SqliteData, Box<dyn std::error::Error>> {
-    read_sqlite_filtered(path, skip_deleted, embedding_dim, config, 0)
-}
-
-/// Like [`read_sqlite`] but only reads chunks whose id is greater than
-/// `min_chunk_id` (0 = all). Used for incremental migration.
-pub fn read_sqlite_filtered(
-    path: &str,
-    skip_deleted: bool,
-    embedding_dim: Option<usize>,
-    config: &SchemaConfig,
-    min_chunk_id: i64,
 ) -> Result<SqliteData, Box<dyn std::error::Error>> {
     let conn = Connection::open(path)?;
 
@@ -221,7 +213,7 @@ pub fn read_sqlite_filtered(
         None => detect_embedding_dim(&conn, config)?.unwrap_or(0),
     };
 
-    let chunks = read_chunks(&conn, skip_deleted, dim, config, min_chunk_id)?;
+    let chunks = read_chunks(&conn, skip_deleted, config)?;
     let sessions = read_sessions(&conn, config)?;
     let entities = read_entities(&conn, config)?;
     let relations = read_relations(&conn, config)?;
@@ -232,42 +224,43 @@ pub fn read_sqlite_filtered(
         entities,
         relations,
         embedding_dim: dim,
-        source_path: path.to_owned(),
     })
 }
 
 fn read_chunks(
     conn: &Connection,
     skip_deleted: bool,
-    expected_dim: usize,
     config: &SchemaConfig,
-    min_chunk_id: i64,
 ) -> SqlResult<Vec<MemoryChunk>> {
     let id_col = config.chunks.columns.first().copied().unwrap_or("id");
     let deleted_col = config.chunks.columns.get(7).copied().unwrap_or("deleted");
-    let mut conds = Vec::new();
+    let mut where_clause = String::new();
     if skip_deleted {
-        conds.push(format!("{deleted_col} = 0"));
+        where_clause = format!(" WHERE {deleted_col} = 0");
     }
-    if min_chunk_id > 0 {
-        conds.push(format!("{id_col} > {min_chunk_id}"));
-    }
-    let where_clause = if conds.is_empty() {
-        String::new()
-    } else {
-        format!(" WHERE {}", conds.join(" AND "))
-    };
+    // In id order, so the store's records follow the source's order.
+    where_clause.push_str(&format!(" ORDER BY {id_col}"));
     let sql = config.chunks.select(&where_clause);
 
     let mut stmt = conn.prepare(&sql)?;
     let rows = stmt.query_map([], |row| {
         let blob: Vec<u8> = row.get(2)?;
-        let mut embedding = blob_to_f32(&blob);
-
-        // Validate/truncate to expected dimension
-        if expected_dim > 0 {
-            embedding.truncate(expected_dim);
+        if !blob.len().is_multiple_of(4) {
+            let id: i64 = row.get(0)?;
+            return Err(rusqlite::Error::FromSqlConversionFailure(
+                2,
+                rusqlite::types::Type::Blob,
+                format!(
+                    "chunk id {id}: embedding BLOB is {} bytes, not a whole number of \
+                     little-endian f32 values",
+                    blob.len()
+                )
+                .into(),
+            ));
         }
+        // Read at full length: rows of the wrong dimension are rejected by
+        // the writer, never truncated to fit.
+        let embedding = blob_to_f32(&blob);
 
         Ok(MemoryChunk {
             id: row.get(0)?,

crates/clawhdf5-migrate/src/store_writer.rs

@@ -0,0 +1,407 @@
+//! Write migrated SQLite data into a clawhdf5-agent store.
+//!
+//! Everything goes through `clawhdf5-agent`'s own API — `HDF5Memory::create`
+//! (or `open` for `--incremental`), `save_batch`, `delete_batch`, the session
+//! cache and the knowledge graph — so the result is an ordinary agent store
+//! that `HDF5Memory::open` accepts, not a second hand-built copy of its schema.
+
+use std::collections::{HashMap, HashSet};
+use std::path::Path;
+
+use clawhdf5_agent::{AgentMemory, HDF5Memory, MemoryConfig, MemoryEntry};
+use clawhdf5_format::float16::round_to_f16;
+
+use crate::sqlite_reader::{MemoryChunk, SqliteData};
+
+type BoxErr = Box<dyn std::error::Error>;
+
+/// SQLite timestamps are Unix seconds; the agent's session and relation
+/// timestamps are Unix microseconds (memory records stay in seconds).
+pub const US_PER_SEC: f64 = 1_000_000.0;
+
+/// Options controlling the output store.
+#[derive(Debug, Clone)]
+pub struct WriteOptions {
+    pub agent_id: String,
+    pub embedder: String,
+    pub compression: bool,
+    pub compression_level: u32,
+    /// Store full-precision `f32` embeddings instead of the library default
+    /// (half precision). Only applies to a newly created store: an existing
+    /// store keeps the precision it was created with.
+    pub f32: bool,
+    /// Add to the store at the output path if there is one, instead of
+    /// replacing it.
+    pub incremental: bool,
+    /// Leave out deleted source rows that are not in the store. (A deleted
+    /// row that matches an active store record still tombstones it, so pass
+    /// deleted rows in `data` for an incremental run.)
+    pub skip_deleted: bool,
+}
+
+/// What the migration wrote, and where each source row went, so validation
+/// can compare the store with the source row by row.
+#[derive(Debug, Default)]
+pub struct Migration {
+    /// Whether the output store existed and was added to (`--incremental`).
+    pub appended_to_existing: bool,
+    /// The store's embedding precision.
+    pub float16: bool,
+    pub embedding_dim: usize,
+    /// Records in the store after the migration (including tombstones).
+    pub store_count: usize,
+    /// `(store index, source chunk index)` of every record written.
+    pub records: Vec<(usize, usize)>,
+    /// Source chunks already in the store (incremental), not written again.
+    pub chunks_present: usize,
+    /// `(store index, source chunk index)` of records that were active in
+    /// the store but whose source row is now deleted (incremental): they were
+    /// tombstoned by this run.
+    pub deleted_in_store: Vec<(usize, usize)>,
+    /// Source rows that were deleted in the store but are active in the
+    /// source (incremental): the agent has no un-delete, so each was written
+    /// again as a new record (counted in `records` too).
+    pub restored: usize,
+    /// Deleted source rows left out because of `skip_deleted`.
+    pub deleted_skipped: usize,
+    /// `(store session index, source session index)` of each session written.
+    pub sessions: Vec<(usize, usize)>,
+    pub sessions_present: usize,
+    /// `(store entity id, source entity index)` of each entity written.
+    pub entities: Vec<(u64, usize)>,
+    pub entities_present: usize,
+    /// SQLite entity id -> store entity id, for every source entity.
+    pub entity_ids: HashMap<i64, u64>,
+    /// `(store relation index, source relation index)` of each relation written.
+    pub relations: Vec<(usize, usize)>,
+    pub relations_present: usize,
+    /// Source relations naming an entity id that is not in the entities
+    /// table; the knowledge graph cannot hold them, so they are skipped.
+    pub dangling_relations: Vec<usize>,
+    /// Messages of the write-anomaly alerts the agent raised while importing
+    /// (informational; they never block a save — a bulk import typically
+    /// trips the write-rate check).
+    pub anomaly_alerts: Vec<String>,
+}
+
+/// Identity of a memory record for incremental de-duplication: every field
+/// the agent stores except the embedding (whose stored form depends on the
+/// store's precision).
+type RecordKey = (String, String, String, String, u64);
+
+fn record_key(
+    chunk: &str,
+    source_channel: &str,
+    session_id: &str,
+    tags: &str,
+    ts: f64,
+) -> RecordKey {
+    (
+        chunk.to_owned(),
+        source_channel.to_owned(),
+        session_id.to_owned(),
+        tags.to_owned(),
+        ts.to_bits(),
+    )
+}
+
+/// Reject rows the agent would otherwise store differently from the source,
+/// or not at all: an embedding of a different length from the store's
+/// dimension (the agent pads/truncates silently), an empty embedding, or, in
+/// a float16 store, a value beyond the half-precision range.
+///
+/// Every source row is checked, including ones that end up not being written
+/// (already in the store, or deleted and skipped): the source must be
+/// consistent as a whole, and the check runs before the store is touched.
+fn check_chunks(chunks: &[MemoryChunk], dim: usize, float16: bool) -> Result<(), BoxErr> {
+    for c in chunks {
+        if c.embedding.is_empty() {
+            return Err(format!(
+                "chunk id {}: the embedding is empty; an agent store needs an embedding \
+                 for every record",
+                c.id
+            )
+            .into());
+        }
+        if c.embedding.len() != dim {
+            return Err(format!(
+                "chunk id {}: embedding has {} values, expected {dim}; every row must have \
+                 the store's dimension (detected from the first row unless --embedding-dim \
+                 is given), and rows are never truncated or padded to fit",
+                c.id,
+                c.embedding.len()
+            )
+            .into());
+        }
+        if float16
+            && let Some((k, v)) = c
+                .embedding
+                .iter()
+                .enumerate()
+                .find(|&(_, &v)| v.is_finite() && round_to_f16(v).is_infinite())
+        {
+            return Err(format!(
+                "chunk id {}: embedding[{k}] = {v} is outside the half-precision range \
+                 (±65504) of a float16 store; migrate with --f32",
+                c.id
+            )
+            .into());
+        }
+    }
+    Ok(())
+}
+
+/// Migrate `data` into the agent store at `path`.
+///
+/// Without `opts.incremental` (or when nothing exists at `path`) a new store
+/// is created, replacing any file there — but only once every source row has
+/// passed [`check_chunks`], so a source that cannot be migrated leaves an
+/// existing store untouched. With it, the existing store is opened and only
+/// source rows it does not already hold are added: memory records are
+/// matched on their content, sessions on their id, entities on name and
+/// type, relations on (source, target, relation). A matched record then
+/// takes the source row's deleted flag: see [`Migration::deleted_in_store`]
+/// and [`Migration::restored`].
+pub fn write_store(
+    path: &Path,
+    data: &SqliteData,
+    opts: &WriteOptions,
+) -> Result<Migration, BoxErr> {
+    let existing = opts.incremental && path.exists();
+    let mut mem = if existing {
+        // `open` does not modify the store beyond what the agent itself does
+        // on open; the checks below run before anything is written.
+        let mem = HDF5Memory::open(path)?;
+        let dim = mem.config().embedding_dim;
+        // `data.embedding_dim` is 0 only for a source with no records and no
+        // --embedding-dim, which has no dimension to disagree with.
+        if data.embedding_dim != 0 && dim != data.embedding_dim {
+            let hint = if dim == 0 {
+                " (a store created from a source with no memory records; re-create it \
+                 with --embedding-dim)"
+            } else {
+                ""
+            };
+            return Err(format!(
+                "the store at {} has embedding_dim {dim}{hint}, the source {}; \
+                 embeddings of a different dimension cannot be added to it",
+                path.display(),
+                data.embedding_dim
+            )
+            .into());
+        }
+        check_chunks(&data.chunks, dim, mem.config().float16)?;
+        mem
+    } else {
+        // (With records, a dimension of 0 means an empty first embedding,
+        // which `check_chunks` reports more precisely.)
+        if data.embedding_dim == 0 && data.chunks.is_empty() {
+            return Err(
+                "the source has no memory records to detect the embedding dimension \
+                        from; pass --embedding-dim (the dimension of the agent's embedder), \
+                        or the store could never hold a record"
+                    .into(),
+            );
+        }
+        let mut config = MemoryConfig::new(path.to_path_buf(), &opts.agent_id, data.embedding_dim);
+        config.embedder = opts.embedder.clone();
+        config.compression = opts.compression;
+        config.compression_level = opts.compression_level;
+        // Only ever switch the library default off (as `clawhdf5-cli create`).
+        if opts.f32 {
+            config.float16 = false;
+        }
+        // Before `create`, which replaces whatever is at `path`.
+        check_chunks(&data.chunks, config.embedding_dim, config.float16)?;
+        HDF5Memory::create(config)?
+    };
+    let float16 = mem.config().float16;
+    let dim = mem.config().embedding_dim;
+
+    let mut m = Migration {
+        appended_to_existing: existing,
+        float16,
+        embedding_dim: dim,
+        ..Migration::default()
+    };
+
+    // ---- Memory records --------------------------------------------------
+    // Store indices of every record the store already holds, by content, so
+    // a source row that appears twice is only treated as present as often
+    // as the store has it.
+    let mut present: HashMap<RecordKey, Vec<usize>> = HashMap::new();
+    if existing {
+        let c = &mem.cache;
+        for i in 0..c.len() {
+            let key = record_key(
+                &c.chunks[i],
+                &c.source_channels[i],
+                &c.session_ids[i],
+                &c.tags[i],
+                c.timestamps[i],
+            );
+            present.entry(key).or_default().push(i);
+        }
+    }
+    let key_of = |c: &MemoryChunk| {
+        record_key(
+            &c.chunk,
+            &c.source_channel,
+            &c.session_id,
+            &c.tags,
+            c.timestamp,
+        )
+    };
+    let tombstoned = |idx: usize| mem.cache.tombstones[idx] != 0;
+    // Pass 1: a store record in the same deleted state as the source row.
+    let mut unmatched: Vec<usize> = Vec::new();
+    for (i, c) in data.chunks.iter().enumerate() {
+        let src_deleted = c.deleted != 0;
+        let hit = present.get_mut(&key_of(c)).and_then(|idxs| {
+            let at = idxs.iter().position(|&x| tombstoned(x) == src_deleted)?;
+            Some(idxs.remove(at))
+        });
+        match hit {
+            Some(_) => m.chunks_present += 1,
+            None => unmatched.push(i),
+        }
+    }
+    // Pass 2: a store record whose deleted state differs — the source row
+    // was deleted or restored since the last migration. The source wins.
+    let mut new_chunks: Vec<usize> = Vec::with_capacity(unmatched.len());
+    let mut delete_in_store: Vec<usize> = Vec::new();
+    for i in unmatched {
+        let c = &data.chunks[i];
+        let hit = present
+            .get_mut(&key_of(c))
+            .and_then(|idxs| (!idxs.is_empty()).then(|| idxs.remove(0)));
+        match hit {
+            // Active in the store, deleted in the source: tombstone it.
+            Some(idx) if c.deleted != 0 => {
+                m.deleted_in_store.push((idx, i));
+                delete_in_store.push(idx);
+            }
+            // Deleted in the store, active in the source. The agent has no
+            // un-delete, so the row is written again as a new active record
+            // (the tombstone stays until the store is compacted).
+            Some(_) => {
+                m.restored += 1;
+                new_chunks.push(i);
+            }
+            None if c.deleted != 0 && opts.skip_deleted => m.deleted_skipped += 1,
+            None => new_chunks.push(i),
+        }
+    }
+    new_chunks.sort_unstable();
+    let to_write: Vec<&MemoryChunk> = new_chunks.iter().map(|&i| &data.chunks[i]).collect();
+
+    // ---- Sessions (in the cache; persisted by the save_batch checkpoint) ---
+    let known_sessions: HashSet<String> = mem
+        .sessions()
+        .entries
+        .iter()
+        .map(|e| e.id.clone())
+        .collect();
+    for (i, s) in data.sessions.iter().enumerate() {
+        if known_sessions.contains(&s.id) {
+            m.sessions_present += 1;
+            continue;
+        }
+        let sessions = mem.sessions_mut();
+        let at = sessions.len();
+        sessions.add_at(
+            &s.id,
+            s.start_idx.max(0) as usize,
+            s.end_idx.max(0) as usize,
+            &s.channel,
+            &s.summary,
+            s.timestamp * US_PER_SEC,
+        );
+        m.sessions.push((at, i));
+    }
+
+    // ---- Knowledge graph -------------------------------------------------
+    let kg = mem.knowledge_mut();
+    // Matched only against what the store held before this run: the source
+    // itself is copied as it is, duplicates included.
+    let by_name_type: HashMap<(String, String), u64> = kg
+        .entities
+        .iter()
+        .map(|e| ((e.name.clone(), e.entity_type.clone()), e.id))
+        .collect();
+    for (i, e) in data.entities.iter().enumerate() {
+        let key = (e.name.clone(), e.entity_type.clone());
+        let id = match by_name_type.get(&key) {
+            Some(&id) => {
+                m.entities_present += 1;
+                id
+            }
+            None => {
+                let id = kg.add_entity(&e.name, &e.entity_type, e.embedding_idx);
+                m.entities.push((id, i));
+                id
+            }
+        };
+        m.entity_ids.insert(e.id, id);
+    }
+    let known_relations: HashSet<(u64, u64, String)> = kg
+        .relations
+        .iter()
+        .map(|r| (r.src, r.tgt, r.relation.clone()))
+        .collect();
+    for (i, r) in data.relations.iter().enumerate() {
+        let (Some(&src), Some(&tgt)) = (m.entity_ids.get(&r.src), m.entity_ids.get(&r.tgt)) else {
+            m.dangling_relations.push(i);
+            continue;
+        };
+        if known_relations.contains(&(src, tgt, r.relation.clone())) {
+            m.relations_present += 1;
+            continue;
+        }
+        let at = kg.relations.len();
+        kg.add_relation(src, tgt, &r.relation, r.weight as f32);
+        kg.relations[at].ts = r.timestamp * US_PER_SEC;
+        m.relations.push((at, i));
+    }
+
+    // ---- Write: one checkpoint for records, sessions and graph -----------
+    let entries: Vec<MemoryEntry> = to_write
+        .iter()
+        .map(|c| MemoryEntry {
+            chunk: c.chunk.clone(),
+            embedding: c.embedding.clone(),
+            source_channel: c.source_channel.clone(),
+            timestamp: c.timestamp,
+            session_id: c.session_id.clone(),
+            tags: c.tags.clone(),
+        })
+        .collect();
+    let indices = mem.save_batch(entries)?;
+    m.records = indices
+        .iter()
+        .copied()
+        .zip(new_chunks.iter().copied())
+        .collect();
+
+    // Rows deleted in the source stay deleted: tombstones, as the agent's own
+    // `delete` leaves them (not compacted away).
+    // Records matched in the store whose source row has since been deleted
+    // are tombstoned too.
+    let tombstones: Vec<usize> = m
+        .records
+        .iter()
+        .filter(|&&(_, src)| data.chunks[src].deleted != 0)
+        .map(|&(idx, _)| idx)
+        .chain(delete_in_store)
+        .collect();
+    mem.delete_batch(&tombstones)?;
+
+    m.anomaly_alerts = mem
+        .take_anomaly_alerts()
+        .into_iter()
+        .map(|a| a.message)
+        .collect();
+    m.store_count = mem.count();
+    drop(mem); // release the single-writer lock before anyone re-opens it
+    Ok(m)
+}

crates/clawhdf5-migrate/src/validate.rs

@@ -1,192 +1,266 @@
-use clawhdf5::reader::File as Hdf5File;
-use clawhdf5_format::provenance::VerifyResult;
+//! Validate a migration by reading the store back the way an agent would:
+//! through `HDF5Memory::open_read_only`, comparing what it loads with the
+//! SQLite source, and running a search for a migrated record.
+
+use std::path::Path;
+
+use clawhdf5_agent::{AgentMemory, HDF5Memory, SearchOptions};
+use clawhdf5_format::float16::round_to_f16;
 
-use crate::hdf5_reader::read_hdf5;
 use crate::sqlite_reader::SqliteData;
+use crate::store_writer::{Migration, US_PER_SEC};
 
 type BoxErr = Box<dyn std::error::Error>;
 
 /// Summary of a migration validation.
 #[derive(Debug)]
 pub struct ValidationSummary {
-    pub chunks: u64,
-    pub sessions: u64,
-    pub entities: u64,
-    pub relations: u64,
-    pub embedding_dim: u64,
-    /// Number of rows whose full content was compared against the source.
+    /// Records in the store (including tombstones).
+    pub count: usize,
+    /// Records in the store that are not deleted.
+    pub active: usize,
+    pub sessions: usize,
+    pub entities: usize,
+    pub relations: usize,
+    pub embedding_dim: usize,
+    pub float16: bool,
+    /// Rows whose full content was compared against the source.
     pub rows_checked: u64,
-    /// Whether the `chunks/text` and `chunks/embeddings` SHINES provenance
-    /// hashes (written via [`crate::hdf5_writer`]) were both present and
-    /// matched their recomputed SHA-256 on read-back. `false` when either
-    /// dataset has no provenance metadata (e.g. an older output file) or
-    /// there are zero chunks to check.
-    pub provenance_verified: bool,
+    /// Whether a search for a migrated record found it (`false` when there
+    /// was no active migrated record with an embedding to search for).
+    pub search_checked: bool,
 }
 
-/// Validate a migrated HDF5 file against the source data.
+/// Validate the store at `path` against the source rows `migration` wrote.
 ///
-/// Reads the written file back and compares actual content — chunk text,
-/// embeddings, and every session/entity/relation field — to the source, not
-/// just the row counts. When `full` is false a representative sample of chunk
-/// rows is content-checked (counts and all other groups are always checked in
-/// full); when `full` is true every chunk row is compared too. `float16` widens
-/// the embedding tolerance to allow for half-precision quantization.
-pub fn validate_hdf5(
-    path: &str,
+/// Counts and the session / entity / relation rows are always checked in
+/// full. Memory records are content-checked on a representative sample, or
+/// all of them with `full`. Embeddings must match exactly: the source values
+/// themselves in an `f32` store, their [`round_to_f16`] in a `float16` one.
+pub fn validate_store(
+    path: &Path,
     source: &SqliteData,
+    migration: &Migration,
     full: bool,
-    float16: bool,
 ) -> Result<ValidationSummary, BoxErr> {
-    let got = read_hdf5(path)?;
-    let provenance_verified = verify_chunk_provenance(path)?;
+    let mut mem = HDF5Memory::open_read_only(path)?;
+    let float16 = mem.config().float16;
+    let dim = mem.config().embedding_dim;
 
     // ---- Counts ----
-    check_count("chunk", got.chunks.len(), source.chunks.len())?;
-    check_count("session", got.sessions.len(), source.sessions.len())?;
-    check_count("entity", got.entities.len(), source.entities.len())?;
-    check_count("relation", got.relations.len(), source.relations.len())?;
-    if got.embedding_dim != source.embedding_dim {
+    check_count("record", mem.count(), migration.store_count)?;
+    if float16 != migration.float16 {
         return Err(format!(
-            "embedding_dim mismatch: HDF5 has {}, source has {}",
-            got.embedding_dim, source.embedding_dim
+            "float16 mismatch: store {float16}, expected {}",
+            migration.float16
         )
         .into());
     }
+    if dim != migration.embedding_dim {
+        return Err(format!(
+            "embedding_dim mismatch: store has {dim}, expected {}",
+            migration.embedding_dim
+        )
+        .into());
+    }
+    if !migration.appended_to_existing {
+        check_count("record", mem.count(), migration.records.len())?;
+        check_count("session", mem.sessions().len(), migration.sessions.len())?;
+        check_count(
+            "entity",
+            mem.knowledge().entities.len(),
+            migration.entities.len(),
+        )?;
+        check_count(
+            "relation",
+            mem.knowledge().relations.len(),
+            migration.relations.len(),
+        )?;
+    }
 
-    // ---- Chunk content (sampled or full) ----
-    let (emb_abs, emb_rel) = if float16 { (1e-2, 1e-2) } else { (1e-4, 0.0) };
+    // ---- Memory records (sampled or full) ----
     let mut rows_checked = 0u64;
-    for i in sample_indices(source.chunks.len(), full) {
-        let (s, g) = (&source.chunks[i], &got.chunks[i]);
-        if s.id != g.id {
-            return Err(field_err("chunk", i, "id", s.id, g.id));
-        }
-        if s.chunk != g.chunk {
-            return Err(format!(
-                "chunk[{i}].text mismatch: source {:?}, HDF5 {:?}",
-                truncate(&s.chunk),
-                truncate(&g.chunk)
-            )
-            .into());
-        }
-        if s.session_id != g.session_id || s.source_channel != g.source_channel || s.tags != g.tags
-        {
-            return Err(format!("chunk[{i}] string field mismatch").into());
-        }
-        if s.deleted != g.deleted {
-            return Err(field_err("chunk", i, "deleted", s.deleted, g.deleted));
-        }
-        if s.embedding.len() != g.embedding.len() {
-            return Err(format!(
-                "chunk[{i}] embedding length mismatch: {} vs {}",
-                s.embedding.len(),
-                g.embedding.len()
-            )
-            .into());
-        }
-        for (k, (&a, &b)) in s.embedding.iter().zip(g.embedding.iter()).enumerate() {
-            if (a - b).abs() > emb_abs + emb_rel * a.abs() {
+    let expected_value = |v: f32| if float16 { round_to_f16(v) } else { v };
+    for k in sample_indices(migration.records.len(), full) {
+        let (idx, src) = migration.records[k];
+        let s = &source.chunks[src];
+        let c = &mem.cache;
+        if idx >= c.len() {
             return Err(
-                    format!("chunk[{i}].embedding[{k}] mismatch: source {a}, HDF5 {b}").into(),
+                format!("record {idx} (chunk id {}) is missing from the store", s.id).into(),
             );
         }
+        let id = s.id;
+        if c.chunks[idx] != s.chunk {
+            return Err(format!(
+                "record {idx} (chunk id {id}) text mismatch: source {:?}, store {:?}",
+                truncate(&s.chunk),
+                truncate(&c.chunks[idx])
+            )
+            .into());
+        }
+        if c.source_channels[idx] != s.source_channel
+            || c.session_ids[idx] != s.session_id
+            || c.tags[idx] != s.tags
+        {
+            return Err(format!("record {idx} (chunk id {id}) string field mismatch").into());
+        }
+        if c.timestamps[idx].to_bits() != s.timestamp.to_bits() {
+            return Err(format!(
+                "record {idx} (chunk id {id}) timestamp mismatch: source {}, store {}",
+                s.timestamp, c.timestamps[idx]
+            )
+            .into());
+        }
+        let deleted = c.tombstones[idx] != 0;
+        if deleted != (s.deleted != 0) {
+            return Err(format!(
+                "record {idx} (chunk id {id}) deleted mismatch: source {}, store {deleted}",
+                s.deleted != 0
+            )
+            .into());
+        }
+        let got = c.embeddings.get(idx).unwrap_or(&[]);
+        if got.len() != s.embedding.len() {
+            return Err(format!(
+                "record {idx} (chunk id {id}) embedding length mismatch: source {}, store {}",
+                s.embedding.len(),
+                got.len()
+            )
+            .into());
+        }
+        for (j, (&a, &b)) in s.embedding.iter().zip(got).enumerate() {
+            let want = expected_value(a);
+            if want.to_bits() != b.to_bits() && !(want.is_nan() && b.is_nan()) {
+                return Err(format!(
+                    "record {idx} (chunk id {id}) embedding[{j}] mismatch: source {a}, \
+                     expected {want}, store {b}"
+                )
+                .into());
+            }
         }
         rows_checked += 1;
     }
 
-    // ---- Other groups (always full — they are small) ----
-    for (i, (s, g)) in source.sessions.iter().zip(got.sessions.iter()).enumerate() {
-        if s.id != g.id
-            || s.start_idx != g.start_idx
-            || s.end_idx != g.end_idx
-            || s.channel != g.channel
-            || s.summary != g.summary
-        {
-            return Err(format!("session[{i}] mismatch").into());
+    // ---- Records tombstoned because their source row was deleted ----
+    for &(idx, src) in &migration.deleted_in_store {
+        let s = &source.chunks[src];
+        let c = &mem.cache;
+        if idx >= c.len() || c.chunks[idx] != s.chunk || c.timestamps[idx] != s.timestamp {
+            return Err(format!("record {idx} (chunk id {}) mismatch or missing", s.id).into());
+        }
+        if c.tombstones[idx] == 0 {
+            return Err(format!(
+                "record {idx} (chunk id {}) is deleted in the source but active in the store",
+                s.id
+            )
+            .into());
         }
         rows_checked += 1;
     }
-    for (i, (s, g)) in source.entities.iter().zip(got.entities.iter()).enumerate() {
-        if s.id != g.id
-            || s.name != g.name
-            || s.entity_type != g.entity_type
-            || s.embedding_idx != g.embedding_idx
+
+    // ---- Sessions ----
+    let sessions = mem.sessions();
+    for &(at, src) in &migration.sessions {
+        let s = &source.sessions[src];
+        let (Some(e), Some(summary)) = (sessions.entries.get(at), sessions.summaries.get(at))
+        else {
+            return Err(format!("session {:?} is missing from the store", s.id).into());
+        };
+        if e.id != s.id
+            || e.start_idx != s.start_idx.max(0) as u64
+            || e.end_idx != s.end_idx.max(0) as u64
+            || e.channel != s.channel
+            || *summary != s.summary
+            || e.ts != s.timestamp * US_PER_SEC
         {
-            return Err(format!("entity[{i}] mismatch").into());
+            return Err(format!("session {:?} mismatch", s.id).into());
         }
         rows_checked += 1;
     }
-    for (i, (s, g)) in source
-        .relations
+
+    // ---- Knowledge graph ----
+    let kg = mem.knowledge();
+    for &(id, src) in &migration.entities {
+        let s = &source.entities[src];
+        let Some(e) = kg.get_entity(id) else {
+            return Err(format!(
+                "entity {:?} (id {}) is missing from the store",
+                s.name, s.id
+            )
+            .into());
+        };
+        if e.name != s.name || e.entity_type != s.entity_type || e.embedding_idx != s.embedding_idx
+        {
+            return Err(format!("entity {:?} (id {}) mismatch", s.name, s.id).into());
+        }
+        rows_checked += 1;
+    }
+    for &(at, src) in &migration.relations {
+        let s = &source.relations[src];
+        let r = kg.relations.get(at);
+        let ok = r.is_some_and(|r| {
+            Some(&r.src) == migration.entity_ids.get(&s.src)
+                && Some(&r.tgt) == migration.entity_ids.get(&s.tgt)
+                && r.relation == s.relation
+                && r.weight == s.weight as f32
+                && r.ts == s.timestamp * US_PER_SEC
+        });
+        if !ok {
+            return Err(format!(
+                "relation {} -[{}]-> {} mismatch or missing",
+                s.src, s.relation, s.tgt
+            )
+            .into());
+        }
+        rows_checked += 1;
+    }
+
+    // ---- A migrated record must be findable by search ----
+    let probe = migration
+        .records
         .iter()
-        .zip(got.relations.iter())
-        .enumerate()
-    {
-        if s.src != g.src || s.tgt != g.tgt || s.relation != g.relation {
-            return Err(format!("relation[{i}] mismatch").into());
+        .copied()
+        .find(|&(idx, _)| dim > 0 && mem.cache.tombstones[idx] == 0);
+    let search_checked = match probe {
+        None => false,
+        Some((idx, _)) => {
+            let query = mem.cache.embeddings[idx].to_vec();
+            let text = mem.cache.chunks[idx].clone();
+            let hits = mem.search(&query, &text, &SearchOptions::new(10));
+            // A record with the same text is as good a hit: the source may
+            // hold duplicates, and they tie.
+            if !hits.iter().any(|h| h.index == idx || h.chunk == text) {
+                return Err(format!(
+                    "search for migrated record {idx} ({:?}) did not return it",
+                    truncate(&text)
+                )
+                .into());
             }
-        rows_checked += 1;
+            true
         }
+    };
 
     Ok(ValidationSummary {
-        chunks: got.chunks.len() as u64,
-        sessions: got.sessions.len() as u64,
-        entities: got.entities.len() as u64,
-        relations: got.relations.len() as u64,
-        embedding_dim: got.embedding_dim as u64,
+        count: mem.count(),
+        active: mem.count_active(),
+        sessions: mem.sessions().len(),
+        entities: mem.knowledge().entities.len(),
+        relations: mem.knowledge().relations.len(),
+        embedding_dim: dim,
+        float16,
         rows_checked,
-        provenance_verified,
+        search_checked,
     })
 }
 
 fn check_count(kind: &str, got: usize, expected: usize) -> Result<(), BoxErr> {
     if got != expected {
-        return Err(format!("{kind} count mismatch: HDF5 has {got}, source has {expected}").into());
+        return Err(format!("{kind} count mismatch: store has {got}, expected {expected}").into());
     }
     Ok(())
 }
 
-/// Re-verify the SHA-256 provenance hash of `chunks/text` and
-/// `chunks/embeddings` against their actual stored bytes, catching
-/// post-write corruption that a plain content comparison against the
-/// in-memory source wouldn't (the source is compared against what
-/// `read_hdf5` decoded, not against the raw bytes on disk).
-///
-/// Returns `Ok(true)` only if both datasets exist and both hashes match.
-/// Returns `Ok(false)` (not an error) if a dataset has no provenance
-/// attributes at all (e.g. a file written before this check existed) or
-/// there are zero chunks. Returns an error only on an actual hash mismatch —
-/// that indicates real corruption.
-fn verify_chunk_provenance(path: &str) -> Result<bool, BoxErr> {
-    let file = Hdf5File::open(path)?;
-    let Ok(chunks) = file.group("chunks") else {
-        return Ok(false);
-    };
-    let mut all_present = true;
-    for name in ["text", "embeddings"] {
-        let Ok(ds) = chunks.dataset(name) else {
-            all_present = false;
-            continue;
-        };
-        match ds.verify_provenance()? {
-            VerifyResult::Ok => {}
-            VerifyResult::NoHash => all_present = false,
-            VerifyResult::Mismatch { stored, computed } => {
-                return Err(format!(
-                    "provenance hash mismatch on chunks/{name}: stored {stored}, recomputed {computed} — data may be corrupted"
-                )
-                .into());
-            }
-        }
-    }
-    Ok(all_present)
-}
-
-fn field_err<T: std::fmt::Display>(kind: &str, i: usize, field: &str, s: T, g: T) -> BoxErr {
-    format!("{kind}[{i}].{field} mismatch: source {s}, HDF5 {g}").into()
-}
-
 fn truncate(s: &str) -> String {
     if s.len() <= 40 {
         s.to_string()
@@ -196,7 +270,7 @@ fn truncate(s: &str) -> String {
     }
 }
 
-/// Indices of chunk rows to content-check. Full = all; otherwise a spread of
+/// Indices of records to content-check. Full = all; otherwise a spread of
 /// representative rows (first/last and evenly-spaced interior samples).
 fn sample_indices(n: usize, full: bool) -> Vec<usize> {
     if n == 0 {

crates/clawhdf5-napi/Cargo.toml

@@ -2,6 +2,7 @@
 name = "clawhdf5-napi"
 version = "2.7.0"
 edition = "2024"
+rust-version.workspace = true
 description = "Node.js native addon (napi-rs) exposing clawhdf5-agent to TypeScript/JavaScript"
 license = "MIT"
 repository = "https://git.redclaw.dev/quantumclaw/clawhdf5"

crates/clawhdf5-netcdf4/Cargo.toml

@@ -2,6 +2,7 @@
 name = "clawhdf5-netcdf4"
 version = "2.7.0"
 edition = "2024"
+rust-version.workspace = true
 description = "NetCDF-4 read support built on rustyhdf5 — pure Rust, no C dependencies"
 license = "MIT"
 repository = "https://git.redclaw.dev/quantumclaw/clawhdf5"

crates/clawhdf5-py/Cargo.toml

@@ -2,6 +2,7 @@
 name = "clawhdf5-py"
 version = "2.7.0"
 edition = "2024"
+rust-version.workspace = true
 description = "Python bindings for rustyhdf5 — a pure-Rust HDF5 library"
 license = "MIT"
 repository = "https://git.redclaw.dev/quantumclaw/clawhdf5"

crates/clawhdf5/Cargo.toml

@@ -2,6 +2,7 @@
 name = "clawhdf5"
 version = "2.7.0"
 edition = "2024"
+rust-version.workspace = true
 description = "Pure-Rust HDF5 reader/writer — no C dependencies"
 license = "MIT"
 repository = "https://git.redclaw.dev/quantumclaw/clawhdf5"
@@ -30,9 +31,10 @@ name = "parallel_bench"
 harness = false
 
 [features]
-default = ["mmap", "fast-deflate", "provenance"]
+default = ["mmap", "provenance"]
 mmap = ["clawhdf5-io/mmap"]
 parallel = ["clawhdf5-format/parallel", "rayon"]
+# zlib-ng (C, needs cmake) instead of the default pure-Rust zlib-rs.
 fast-deflate = ["clawhdf5-format/fast-deflate"]
 apple-compression = []
 zstd = ["clawhdf5-format/zstd"]

crates/clawhdf5/src/reader.rs

@@ -478,7 +478,12 @@ impl<'f> Dataset<'f> {
         // sparse) dataset — select from a fill-aware full read instead. (The
         // selection reader currently decodes the full dataset too, so this
         // costs nothing extra.)
-        let fill = clawhdf5_format::fill_value::dataset_fill_value(&self.header.messages)?;
+        let fill = clawhdf5_format::fill_value::dataset_fill_value_in(
+            self.file.data.as_bytes(),
+            &self.header.messages,
+            self.file.offset_size(),
+            self.file.length_size(),
+        )?;
         let fill_matters = !clawhdf5_format::fill_value::has_storage(&dl)
             || (matches!(dl, DataLayout::Chunked { .. })
                 && !clawhdf5_format::fill_value::is_default(fill.as_deref()));

crates/clawhdf5/tests/chunk_index_interop.rs

@@ -0,0 +1,593 @@
+//! Fixed Array / Extensible Array chunk-index interop with libhdf5 (via h5py).
+//!
+//! Both indexes place each chunk at a linear index computed from the
+//! dataset's *maximum* dimensions, and the Extensible Array additionally
+//! moves its unlimited dimension to the slowest-varying position. Getting
+//! either wrong reads (or writes) every chunk after the first row in the
+//! wrong place, silently, so these tests compare every value.
+//!
+//! Skipped when python3 with h5py is unavailable, unless
+//! `CLAWHDF5_REQUIRE_INTEROP=1`.
+
+use std::process::Command;
+
+use clawhdf5::{File, FileBuilder};
+
+fn python() -> String {
+    std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
+}
+
+fn interop_required() -> bool {
+    std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
+}
+
+fn python_available() -> bool {
+    Command::new(python())
+        .args(["-c", "import h5py"])
+        .output()
+        .map(|o| o.status.success())
+        .unwrap_or(false)
+}
+
+macro_rules! skip_if_no_python {
+    () => {
+        if !python_available() {
+            assert!(
+                !interop_required(),
+                "CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available"
+            );
+            eprintln!("SKIP: python3 with h5py not available");
+            return;
+        }
+    };
+}
+
+fn run_python(script: &str) -> String {
+    let output = Command::new(python())
+        .args(["-c", script])
+        .output()
+        .expect("failed to run python");
+    if !output.status.success() {
+        panic!(
+            "Python script failed:\nSTDOUT: {}\nSTDERR: {}",
+            String::from_utf8_lossy(&output.stdout),
+            String::from_utf8_lossy(&output.stderr)
+        );
+    }
+    String::from_utf8_lossy(&output.stdout).trim().to_string()
+}
+
+/// Row-major `arange` of `shape`, cropped to `crop` (the current extent).
+fn arange_cropped(full: &[usize], crop: &[usize]) -> Vec<i32> {
+    let n: usize = crop.iter().product();
+    let mut out = Vec::with_capacity(n);
+    for flat in 0..n {
+        let mut rem = flat;
+        let mut src = 0usize;
+        let mut stride = 1usize;
+        let mut coords = vec![0usize; crop.len()];
+        for d in (0..crop.len()).rev() {
+            coords[d] = rem % crop[d];
+            rem /= crop[d];
+        }
+        for d in (0..full.len()).rev() {
+            src += coords[d] * stride;
+            stride *= full[d];
+        }
+        out.push(src as i32);
+    }
+    out
+}
+
+/// One `i4` dataset, filled with `arange` over `full` and then resized to
+/// `shape` (equal to `full` unless the case shrinks it).
+struct Case {
+    name: &'static str,
+    full: Vec<usize>,
+    shape: Vec<usize>,
+    chunks: Vec<usize>,
+    maxshape: &'static str,
+    extra: &'static str,
+    index: &'static str,
+}
+
+fn py_tuple(v: &[usize]) -> String {
+    let parts: Vec<String> = v.iter().map(|x| x.to_string()).collect();
+    format!("({},)", parts.join(","))
+}
+
+/// Have h5py (`libver="latest"`, so Fixed/Extensible Array indexes) write
+/// every case to one file, then read each back and compare every value.
+fn check_h5py_written(cases: &[Case]) {
+    let dir = tempfile::tempdir().unwrap();
+    let path = dir.path().join("h5py_chunk_index.h5");
+    let path_str = path.display().to_string();
+
+    let mut script =
+        format!("import h5py, numpy as np\nf = h5py.File(r'{path_str}', 'w', libver='latest')\n");
+    for c in cases {
+        script += &format!(
+            "d = f.create_dataset('{name}', data=np.arange({n}, dtype='i4').reshape({full}), \
+             chunks={chunks}, maxshape={maxshape}{extra})\n\
+             d.resize({shape})\n",
+            name = c.name,
+            n = c.full.iter().product::<usize>(),
+            full = py_tuple(&c.full),
+            chunks = py_tuple(&c.chunks),
+            maxshape = c.maxshape,
+            extra = c.extra,
+            shape = py_tuple(&c.shape),
+        );
+    }
+    script += "f.close()\n";
+    run_python(&script);
+
+    let file = File::open(&path).unwrap();
+    for c in cases {
+        let ds = file.dataset(c.name).unwrap();
+        let shape: Vec<usize> = ds.shape().unwrap().iter().map(|&d| d as usize).collect();
+        assert_eq!(shape, c.shape, "{}: shape", c.name);
+        let got = ds.read_i32().unwrap();
+        let want = arange_cropped(&c.full, &c.shape);
+        let bad = got.iter().zip(&want).filter(|(a, b)| a != b).count();
+        assert_eq!(
+            got,
+            want,
+            "{}: {bad} of {} values differ (index {})",
+            c.name,
+            want.len(),
+            c.index
+        );
+    }
+}
+
+/// h5py-written Extensible Array whose unlimited dimension is not the first,
+/// with the current shape smaller than the finite maximum: the library
+/// swizzles the unlimited dimension to the slowest position and strides the
+/// rest by their maximum chunk counts.
+#[test]
+fn h5py_extensible_array_partial_extent_reads_correctly() {
+    skip_if_no_python!();
+    check_h5py_written(&[
+        // The `ea_fa_partial.h5` repro from the conformance sweep.
+        Case {
+            name: "ea_10_none",
+            full: vec![4, 6],
+            shape: vec![4, 6],
+            chunks: vec![2, 3],
+            maxshape: "(10, None)",
+            extra: "",
+            index: "EA, unlimited dim 1",
+        },
+        Case {
+            name: "ea_none_10",
+            full: vec![4, 6],
+            shape: vec![4, 6],
+            chunks: vec![2, 3],
+            maxshape: "(None, 10)",
+            extra: "",
+            index: "EA, unlimited dim 0",
+        },
+        Case {
+            name: "ea_3d_mid",
+            full: vec![3, 4, 5],
+            shape: vec![3, 4, 5],
+            chunks: vec![2, 3, 2],
+            maxshape: "(5, None, 7)",
+            extra: "",
+            index: "EA, unlimited dim 1 of 3",
+        },
+        Case {
+            name: "ea_3d_last_gzip",
+            full: vec![3, 4, 5],
+            shape: vec![3, 4, 5],
+            chunks: vec![2, 3, 2],
+            maxshape: "(5, 9, None)",
+            extra: ", compression='gzip'",
+            index: "EA, unlimited dim 2 of 3, filtered",
+        },
+        // Many chunks: crosses data blocks, super blocks and paging.
+        Case {
+            name: "ea_many",
+            full: vec![3, 1500],
+            shape: vec![3, 1500],
+            chunks: vec![1, 1],
+            maxshape: "(4, None)",
+            extra: "",
+            index: "EA, 4500 slots",
+        },
+        // Shrunk after writing: chunks beyond the extent must be ignored.
+        Case {
+            name: "ea_shrunk",
+            full: vec![8, 9],
+            shape: vec![3, 4],
+            chunks: vec![2, 3],
+            maxshape: "(10, None)",
+            extra: "",
+            index: "EA, shrunk",
+        },
+    ]);
+}
+
+/// h5py-written Fixed Array with the current shape smaller than a finite
+/// maxshape: the index has one slot per chunk of the *maximum* extent.
+#[test]
+fn h5py_fixed_array_partial_extent_reads_correctly() {
+    skip_if_no_python!();
+    check_h5py_written(&[
+        Case {
+            name: "fa_20_10",
+            full: vec![4, 6],
+            shape: vec![4, 6],
+            chunks: vec![2, 3],
+            maxshape: "(20, 10)",
+            extra: "",
+            index: "FA",
+        },
+        Case {
+            name: "fa_3d_gzip",
+            full: vec![3, 4, 5],
+            shape: vec![3, 4, 5],
+            chunks: vec![2, 3, 2],
+            maxshape: "(6, 8, 10)",
+            extra: ", compression='gzip'",
+            index: "FA, filtered",
+        },
+        // Paged (> 1024 slots) with most of them beyond the extent.
+        Case {
+            name: "fa_paged",
+            full: vec![30, 50],
+            shape: vec![30, 50],
+            chunks: vec![1, 1],
+            maxshape: "(40, 60)",
+            extra: "",
+            index: "FA, 2400 slots, paged",
+        },
+        Case {
+            name: "fa_shrunk",
+            full: vec![8, 9],
+            shape: vec![5, 2],
+            chunks: vec![2, 3],
+            maxshape: "(20, 10)",
+            extra: "",
+            index: "FA, shrunk",
+        },
+    ]);
+}
+
+// ===========================================================================
+// Files we write, read back by libhdf5 (h5py and h5dump) and by us
+// ===========================================================================
+
+/// One `i4` dataset we write, filled with `arange` over `shape`.
+struct WriteCase {
+    name: String,
+    shape: Vec<u64>,
+    chunks: Vec<u64>,
+    maxshape: Option<Vec<u64>>,
+    deflate: bool,
+}
+
+fn wcase(name: &str, shape: &[u64], chunks: &[u64], maxshape: Option<&[u64]>) -> WriteCase {
+    WriteCase {
+        name: name.to_string(),
+        shape: shape.to_vec(),
+        chunks: chunks.to_vec(),
+        maxshape: maxshape.map(<[u64]>::to_vec),
+        deflate: false,
+    }
+}
+
+fn h5dump_available() -> bool {
+    Command::new("h5dump")
+        .arg("--version")
+        .output()
+        .map(|o| o.status.success())
+        .unwrap_or(false)
+}
+
+/// Write every case into one file with our writer, then check that our own
+/// reader, h5py and h5dump (when installed) all return every value. Only the
+/// libhdf5 half is skipped without h5py.
+fn check_we_write(cases: &[WriteCase]) {
+    let dir = tempfile::tempdir().unwrap();
+    let path = dir.path().join("ours_chunk_index.h5");
+    let path_str = path.display().to_string();
+
+    let mut b = FileBuilder::new();
+    for c in cases {
+        let n: u64 = c.shape.iter().product();
+        let data: Vec<i32> = (0..n as i32).collect();
+        let ds = b.create_dataset(&c.name);
+        ds.with_i32_data(&data)
+            .with_shape(&c.shape)
+            .with_chunks(&c.chunks);
+        if let Some(ms) = &c.maxshape {
+            ds.with_maxshape(ms);
+        }
+        if c.deflate {
+            ds.with_deflate(4);
+        }
+    }
+    b.write(&path).unwrap();
+
+    // Our reader.
+    let file = File::open(&path).unwrap();
+    for c in cases {
+        let got = file.dataset(&c.name).unwrap().read_i32().unwrap();
+        let n: u64 = c.shape.iter().product();
+        let bad = got
+            .iter()
+            .enumerate()
+            .filter(|&(i, &v)| v != i as i32)
+            .count();
+        assert!(
+            got.len() == n as usize && bad == 0,
+            "{}: our reader: {bad} of {n} values wrong",
+            c.name
+        );
+    }
+
+    // libhdf5 via h5py.
+    skip_if_no_python!();
+    let mut script =
+        format!("import h5py, numpy as np\nbad = []\nf = h5py.File(r'{path_str}', 'r')\n");
+    for c in cases {
+        let shape: Vec<String> = c.shape.iter().map(u64::to_string).collect();
+        let maxshape: Vec<String> = c
+            .maxshape
+            .as_ref()
+            .unwrap_or(&c.shape)
+            .iter()
+            .map(|&d| {
+                if d == u64::MAX {
+                    "None".to_string()
+                } else {
+                    d.to_string()
+                }
+            })
+            .collect();
+        script += &format!(
+            "d = f['{name}']\n\
+             want = np.arange({n}, dtype='i4').reshape(({shape},))\n\
+             got = d[()]\n\
+             if d.maxshape != ({maxshape},): bad.append(('{name}', 'maxshape', d.maxshape))\n\
+             elif not np.array_equal(got, want): \
+             bad.append(('{name}', int((got != want).sum()), 'of', got.size))\n",
+            name = c.name,
+            n = c.shape.iter().product::<u64>(),
+            shape = shape.join(","),
+            maxshape = maxshape.join(","),
+        );
+    }
+    script += "print(bad if bad else 'OK')\n";
+    let out = run_python(&script);
+    assert_eq!(out, "OK", "h5py disagrees");
+
+    // libhdf5's own tool, when installed.
+    if h5dump_available() {
+        let o = Command::new("h5dump").arg(&path).output().unwrap();
+        let stderr = String::from_utf8_lossy(&o.stderr);
+        assert!(
+            o.status.success() && !stderr.to_lowercase().contains("error"),
+            "h5dump failed: {stderr}"
+        );
+    }
+
+    // Let libhdf5 grow every resizable dataset by two chunks per dimension
+    // (capped at the maxshape) and rewrite it, which updates our index in
+    // place and inserts new chunks into it. Then both readers must agree.
+    let script = format!(
+        r#"
+import h5py, numpy as np
+grown = {{}}
+with h5py.File(r'{path_str}', 'r+') as f:
+    for name in f:
+        d = f[name]
+        if d.chunks is None:
+            continue
+        new = tuple(s + 2 * c if m is None else min(m, s + 2 * c)
+                    for s, m, c in zip(d.shape, d.maxshape, d.chunks))
+        if new == d.shape:
+            continue
+        old = d[()]
+        full = np.full(new, -7, 'i4')
+        full[tuple(slice(0, s) for s in old.shape)] = old
+        d.resize(new)
+        d[...] = full
+        grown[name] = (list(old.shape), list(new))
+with h5py.File(r'{path_str}', 'r') as f:
+    for name, (old, new) in grown.items():
+        want = np.full(new, -7, 'i4')
+        want[tuple(slice(0, s) for s in old)] = np.arange(int(np.prod(old)), dtype='i4').reshape(old)
+        assert np.array_equal(f[name][()], want), name
+for name, (old, new) in grown.items():
+    print(name, ','.join(map(str, old)), ','.join(map(str, new)))
+"#
+    );
+    let out = run_python(&script);
+    let growable = cases
+        .iter()
+        .filter(|c| c.maxshape.as_ref().is_some_and(|m| *m != c.shape))
+        .count();
+    assert_eq!(out.lines().count(), growable, "libhdf5 grew: {out}");
+    let dims = |s: &str| -> Vec<usize> { s.split(',').map(|x| x.parse().unwrap()).collect() };
+    let file = File::open(&path).unwrap();
+    for line in out.lines() {
+        let mut parts = line.split(' ');
+        let (name, old, new) = (
+            parts.next().unwrap(),
+            dims(parts.next().unwrap()),
+            dims(parts.next().unwrap()),
+        );
+        let got = file.dataset(name).unwrap().read_i32().unwrap();
+        let n: usize = new.iter().product();
+        let mut want = vec![-7i32; n];
+        for (flat, w) in want.iter_mut().enumerate() {
+            let mut rem = flat;
+            let mut coords = vec![0usize; new.len()];
+            for d in (0..new.len()).rev() {
+                coords[d] = rem % new[d];
+                rem /= new[d];
+            }
+            if coords.iter().zip(&old).all(|(c, o)| c < o) {
+                *w = coords.iter().zip(&old).fold(0, |acc, (c, o)| acc * o + c) as i32;
+            }
+        }
+        let bad = got.iter().zip(&want).filter(|(a, b)| a != b).count();
+        assert!(
+            got.len() == n && bad == 0,
+            "{name}: after libhdf5 grew it, our reader got {bad} of {n} values wrong"
+        );
+    }
+}
+
+/// A Fixed Array with more than 1024 elements must be paged, or libhdf5
+/// rejects the data block's checksum.
+#[test]
+fn we_write_paged_fixed_array() {
+    let mut cases: Vec<WriteCase> = [1023u64, 1024, 1025, 2048, 5000]
+        .iter()
+        .map(|&n| wcase(&format!("fa_{n}"), &[n * 4], &[4], None))
+        .collect();
+    // Filtered elements are wider; a 2-D grid pages the same way.
+    let mut filtered = wcase("fa_1500_deflate", &[1500 * 4], &[4], None);
+    filtered.deflate = true;
+    cases.push(filtered);
+    cases.push(wcase("fa_2d_1100", &[110, 40], &[1, 4], None));
+    check_we_write(&cases);
+}
+
+/// An Extensible Array holds 4 elements in its index block and 240 in the
+/// data blocks the index block addresses; everything after that lives under
+/// super blocks, and from ~131K elements on in paged data blocks. Chunks past
+/// index 243 used to be written but never indexed (read back as fill by us
+/// and by libhdf5).
+#[test]
+fn we_write_extensible_array_past_index_block() {
+    let unl: &[u64] = &[u64::MAX];
+    let mut cases: Vec<WriteCase> = [1u64, 4, 5, 243, 244, 245, 300, 1000, 5000]
+        .iter()
+        .map(|&n| wcase(&format!("ea_{n}"), &[n * 4], &[4], Some(unl)))
+        .collect();
+    let mut filtered = wcase("ea_300_deflate", &[300 * 4], &[4], Some(unl));
+    filtered.deflate = true;
+    cases.push(filtered);
+    // Several super blocks and paged data blocks (level 13, the first with
+    // data blocks over 1024 elements, starts at element 4 + 131056).
+    cases.push(wcase("ea_140000", &[140_000], &[1], Some(unl)));
+    check_we_write(&cases);
+}
+
+/// A maxshape larger than the shape: the index must be laid out over the
+/// chunks of the maximum extent (libhdf5 read our Fixed Array past its end:
+/// "addr overflow"), and an Extensible Array whose unlimited dimension is not
+/// the first must swizzle it to the slowest position (libhdf5 read our
+/// `(20, None)` dataset scrambled).
+#[test]
+fn we_write_maxshape_larger_than_shape() {
+    const U: u64 = u64::MAX;
+    let mut cases = vec![
+        // Fixed Array over the maximum extent.
+        wcase("fa2d_finite_max", &[20, 30], &[5, 5], Some(&[40, 60])),
+        wcase("fa1d_finite_max", &[40], &[4], Some(&[100])),
+        wcase("fa3d_edges", &[6, 7, 8], &[4, 3, 5], Some(&[10, 9, 20])),
+        wcase("fa_paged_max", &[30, 50], &[1, 1], Some(&[40, 60])),
+        wcase("fa_one_chunk_now", &[5], &[5], Some(&[50])),
+        // Extensible Array, unlimited dimension first (no swizzle) ...
+        wcase("ea2d_unl_fin", &[20, 30], &[5, 5], Some(&[U, 30])),
+        wcase("ea2d_unl_fin_max", &[20, 30], &[5, 5], Some(&[U, 60])),
+        // ... and not first (swizzled).
+        wcase("ea2d_fin_unl", &[20, 30], &[5, 5], Some(&[20, U])),
+        wcase("ea2d_fin_max_unl", &[20, 30], &[5, 5], Some(&[40, U])),
+        wcase("ea3d_mid", &[6, 7, 8], &[4, 3, 5], Some(&[10, U, 20])),
+        // Past the index block and into super blocks, swizzled.
+        wcase("ea2d_many", &[3, 2000], &[1, 1], Some(&[4, U])),
+    ];
+    let mut filtered = wcase(
+        "ea3d_last_deflate",
+        &[6, 7, 8],
+        &[4, 3, 5],
+        Some(&[6, 8, U]),
+    );
+    filtered.deflate = true;
+    cases.push(filtered);
+    check_we_write(&cases);
+}
+
+/// More than one unlimited dimension needs a version-2 B-tree chunk index,
+/// as the library uses; an Extensible Array for `(None, None)` made libhdf5
+/// refuse the whole file ("already found unlimited dimension").
+#[test]
+fn we_write_btree_v2_for_several_unlimited_dims() {
+    const U: u64 = u64::MAX;
+    let mut cases = vec![
+        wcase("unl_unl", &[20, 30], &[5, 5], Some(&[U, U])),
+        wcase("unl_fin_unl", &[6, 7, 8], &[4, 3, 5], Some(&[U, 9, U])),
+        // More records than the library's 2048-byte node holds (84 here).
+        wcase("unl_unl_2400", &[40, 60], &[1, 1], Some(&[U, U])),
+        wcase("unl_unl_empty", &[0, 0], &[4, 4], Some(&[U, U])),
+    ];
+    let mut filtered = wcase("unl_unl_deflate", &[6, 7, 8], &[4, 3, 5], Some(&[U, U, U]));
+    filtered.deflate = true;
+    cases.push(filtered);
+    check_we_write(&cases);
+}
+
+/// A single-leaf B-tree has a 16-bit record count; beyond it the writer
+/// refuses rather than writing a tree libhdf5 would misread.
+#[test]
+fn btree_v2_index_past_one_leaf_is_refused() {
+    let mut b = FileBuilder::new();
+    b.create_dataset("d")
+        .with_i32_data(&vec![0i32; 70_000])
+        .with_shape(&[70_000, 1])
+        .with_chunks(&[1, 1])
+        .with_maxshape(&[u64::MAX, u64::MAX]);
+    let dir = tempfile::tempdir().unwrap();
+    assert!(b.write(dir.path().join("too_many.h5")).is_err());
+}
+
+/// A maxshape equal to the shape cannot grow, so it needs no chunks: the
+/// dataset stays contiguous (as h5py makes it) unless chunks are requested.
+#[test]
+fn maxshape_equal_to_shape_stays_contiguous() {
+    let dir = tempfile::tempdir().unwrap();
+    let path = dir.path().join("ms_eq.h5");
+    let data: Vec<i32> = (0..40).collect();
+    let mut b = FileBuilder::new();
+    b.create_dataset("plain")
+        .with_i32_data(&data)
+        .with_shape(&[40])
+        .with_maxshape(&[40]);
+    b.create_dataset("chunked")
+        .with_i32_data(&data)
+        .with_shape(&[40])
+        .with_maxshape(&[40])
+        .with_chunks(&[8]);
+    b.write(&path).unwrap();
+
+    let file = File::open(&path).unwrap();
+    let plain = file.dataset("plain").unwrap();
+    assert_eq!(plain.read_i32().unwrap(), data);
+    assert_eq!(plain.max_dimensions().unwrap(), Some(vec![40]));
+    assert!(
+        plain.read_raw_ref().unwrap().is_some(),
+        "maxshape == shape should be contiguous"
+    );
+    let chunked = file.dataset("chunked").unwrap();
+    assert_eq!(chunked.read_i32().unwrap(), data);
+    assert!(chunked.read_raw_ref().unwrap().is_none());
+
+    skip_if_no_python!();
+    let out = run_python(&format!(
+        "import h5py, numpy as np\n\
+         f = h5py.File(r'{}', 'r')\n\
+         for n in ('plain', 'chunked'):\n\
+         \x20   d = f[n]\n\
+         \x20   assert np.array_equal(d[()], np.arange(40, dtype='i4')), n\n\
+         \x20   print(n, d.chunks, d.maxshape)\n",
+        path.display()
+    ));
+    assert_eq!(out, "plain None (40,)\nchunked (8,) (40,)");
+}

crates/clawhdf5/tests/concurrent_chunk_cache.rs

@@ -0,0 +1,87 @@
+//! A `File` is `Send + Sync` and keeps one chunk cache for all its datasets.
+//! Threads reading different chunked datasets through the same `File` must
+//! each get their own dataset's data.
+
+use std::sync::Arc;
+
+use clawhdf5::{File, FileBuilder};
+
+const DATASETS: usize = 24;
+const THREADS: usize = 16;
+const ROUNDS: usize = 40;
+
+/// Contents of dataset `k`: distinct from every other dataset's, element for
+/// element, so any chunk served from the wrong dataset shows.
+fn values(k: usize, n: usize) -> Vec<f64> {
+    (0..n).map(|i| (k * 100_000 + i) as f64).collect()
+}
+
+fn build() -> File {
+    let mut b = FileBuilder::new();
+    for k in 0..DATASETS {
+        let ds = b.create_dataset(&format!("d{k:02}"));
+        match k % 3 {
+            // 1-D, compressed: chunk offsets 0, 8, 16, ... in every dataset.
+            0 => {
+                ds.with_f64_data(&values(k, 64)).with_shape(&[64]);
+                ds.with_chunks(&[8]).with_deflate(1);
+            }
+            // 1-D, shuffle + compressed, a different length.
+            1 => {
+                ds.with_f64_data(&values(k, 40)).with_shape(&[40]);
+                ds.with_chunks(&[8]).with_shuffle().with_deflate(1);
+            }
+            // 2-D, compressed: coordinates (0,0), (0,4), (4,0), ... overlap
+            // the other datasets' in the first dimension.
+            _ => {
+                ds.with_f64_data(&values(k, 64)).with_shape(&[8, 8]);
+                ds.with_chunks(&[4, 4]).with_deflate(1);
+            }
+        }
+    }
+    File::from_bytes(b.finish().unwrap()).unwrap()
+}
+
+fn expected(k: usize) -> Vec<f64> {
+    values(k, if k % 3 == 1 { 40 } else { 64 })
+}
+
+#[test]
+fn threads_reading_different_datasets_get_their_own_chunks() {
+    let file = Arc::new(build());
+    // Sequential sanity check first.
+    for k in 0..DATASETS {
+        let got = file.dataset(&format!("d{k:02}")).unwrap().read_f64();
+        assert_eq!(got.unwrap(), expected(k), "sequential d{k:02}");
+    }
+
+    let handles: Vec<_> = (0..THREADS)
+        .map(|t| {
+            let file = Arc::clone(&file);
+            std::thread::spawn(move || {
+                let mut wrong = Vec::new();
+                for round in 0..ROUNDS {
+                    let k = (t * 7 + round * 5) % DATASETS;
+                    let name = format!("d{k:02}");
+                    match file.dataset(&name).unwrap().read_f64() {
+                        Ok(v) if v == expected(k) => {}
+                        Ok(v) => wrong.push(format!("{name}: wrong data, first {:?}", &v[..4])),
+                        Err(e) => wrong.push(format!("{name}: {e}")),
+                    }
+                }
+                wrong
+            })
+        })
+        .collect();
+    let failures: Vec<String> = handles
+        .into_iter()
+        .flat_map(|h| h.join().unwrap())
+        .collect();
+    assert!(
+        failures.is_empty(),
+        "{} of {} concurrent reads were wrong, e.g. {:?}",
+        failures.len(),
+        THREADS * ROUNDS,
+        &failures[..failures.len().min(5)]
+    );
+}