From 18dc35f7e5c407feffb31c32fbae7cd935a8ce23 Mon Sep 17 00:00:00 2001 From: osobh Date: Sun, 20 Sep 2026 06:01:31 -0700 Subject: [PATCH] chore(release): v2.6.0 Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 99 ++++++++++++++++------------- Cargo.toml | 2 +- crates/clawhdf5-accel/Cargo.toml | 2 +- crates/clawhdf5-agent/Cargo.toml | 14 ++-- crates/clawhdf5-android/Cargo.toml | 2 +- crates/clawhdf5-ann/Cargo.toml | 8 +-- crates/clawhdf5-bench/Cargo.toml | 2 +- crates/clawhdf5-cli/Cargo.toml | 4 +- crates/clawhdf5-derive/Cargo.toml | 2 +- crates/clawhdf5-filters/Cargo.toml | 2 +- crates/clawhdf5-format/Cargo.toml | 4 +- crates/clawhdf5-gpu/Cargo.toml | 2 +- crates/clawhdf5-io/Cargo.toml | 4 +- crates/clawhdf5-migrate/Cargo.toml | 8 +-- crates/clawhdf5-napi/Cargo.toml | 4 +- crates/clawhdf5-netcdf4/Cargo.toml | 6 +- crates/clawhdf5-py/Cargo.toml | 6 +- crates/clawhdf5-py/pyproject.toml | 2 +- crates/clawhdf5/Cargo.toml | 12 ++-- packages/clawhdf5-node/package.json | 2 +- 20 files changed, 98 insertions(+), 89 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d6091c..7ec2f47 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,53 +1,19 @@ # Changelog -## Unreleased +## v2.6.0 (2026-09-20) -### Testing -- The Python interop suites honour **`CLAWHDF5_PYTHON`**, and `ci-test.sh` - picks up a `.venv/bin/python` automatically. On a PEP 668 "externally - managed" system h5py cannot be installed into the system interpreter at all, - so every interop suite — the h5py writer round-trips, the facade, netCDF4 - and the reference files — was skipping silently. A silent skip here is - exactly how the v5 compound-datatype bug reached a release. - `CLAWHDF5_REQUIRE_INTEROP=1` still turns a skip into a failure. - -### Memory -- `clawhdf5-agent`: **`MemoryConfig::quantized_index`** stores the vector - index's own copy of the embeddings as `i8` rather than `f32`, which at 100k - 384-dim entries takes the index from 266 to 123 MiB and the whole reopened - store from 399 to 256 MiB (2.72x -> **1.74x** the raw vectors). Quantised - distances are approximate and `ef` cannot compensate — recall@10 tops out at - 0.967 against f32's 0.9995 — so the query path re-scores the candidate pool - against the exact embeddings the store already holds, which restores recall - (0.9940 vs 0.9945 at ef=64) for about 13% of QPS. **Off by default**: it - trades query speed for memory, and which side is worth more depends on the - deployment. The setting is persisted, so a reopened store does not silently - revert to four times the index memory. -- `clawhdf5-ann`: `Storage::Int8` and the `build_with` / `new_with` / - `from_graph_bytes_with` constructors that select it. The scale is per row, - not global — a fixed `[-1, 1]` scale spends fewer than 12 of the 255 levels - on a unit-length 128-dim vector and is unusable (0.35 top-10 overlap against - an exact ranking, versus 0.99 per row). `compact()` keeps the storage it was - given; serialized indexes still carry f32 vectors, so a quantised index is - rebuilt rather than loaded. - -### Memory -- `clawhdf5-agent`: **a loaded store holds ~30% less memory** (100k 384-dim - entries: 505 -> 357 MiB, 3.44x -> 2.43x the raw vectors). The cache kept - every embedding twice — a `Vec>` and a flattened copy for the - batched kernels, maintained in lock-step — so it now stores only the flat - buffer and indexes into it. Recall and query latency are unchanged. - **Breaking:** `MemoryCache::embeddings` is a `cache::Embeddings` rather than +### Upgrade Notes +- **Re-ranked results change, substantially for the better.** `RerankInput` + and `ReRankConfig` gained fields (`relevance`, `relevance_weight`), so + literal constructions need updating; `..Default::default()` does not. Any + caller that re-ranked was previously getting results ordered by age with the + retrieval score discarded — see below. +- **Breaking:** `MemoryCache::embeddings` is a `cache::Embeddings` rather than a `Vec>` (indexing still yields a `&[f32]` row); `embeddings_flat` is gone, replaced by `flat_embeddings()`; `rebuild_flat()` is a deprecated - no-op. Rows are now always exactly `dim` long — shorter ones are - zero-padded — which makes the ragged-row case that used to silently - misalign the flattened copy unrepresentable. -- `clawhdf5-bench`: `search_harness --footprint` reports live heap use per - stage, measured with a counting allocator (RSS cannot see a structure freed - into the allocator's own pool). - -## Unreleased + no-op. +- `MemoryConfig` gained `quantized_index` (default `false`, so behaviour is + unchanged unless you opt in); literal constructions need the field. ### Retrieval quality - `clawhdf5-agent`: **re-ranking discarded the retrieval score.** @@ -70,6 +36,49 @@ one it supersedes? Plain recall cannot see this, because both are labelled gold. New `--rerank-sweep`. +### Memory +- `clawhdf5-agent`: **`MemoryConfig::quantized_index`** stores the vector + index's own copy of the embeddings as `i8` rather than `f32`, which at 100k + 384-dim entries takes the index from 266 to 123 MiB and the whole reopened + store from 399 to 256 MiB (2.72x -> **1.74x** the raw vectors). Quantised + distances are approximate and `ef` cannot compensate — recall@10 tops out at + 0.967 against f32's 0.9995 — so the query path re-scores the candidate pool + against the exact embeddings the store already holds, which restores recall + (0.9940 vs 0.9945 at ef=64) for about 13% of QPS. **Off by default**: it + trades query speed for memory, and which side is worth more depends on the + deployment. The setting is persisted, so a reopened store does not silently + revert to four times the index memory. +- `clawhdf5-ann`: `Storage::Int8` and the `build_with` / `new_with` / + `from_graph_bytes_with` constructors that select it. The scale is per row, + not global — a fixed `[-1, 1]` scale spends fewer than 12 of the 255 levels + on a unit-length 128-dim vector and is unusable (0.35 top-10 overlap against + an exact ranking, versus 0.99 per row). `compact()` keeps the storage it was + given; serialized indexes still carry f32 vectors, so a quantised index is + rebuilt rather than loaded. +- `clawhdf5-agent`: **a loaded store holds ~30% less memory** (100k 384-dim + entries: 505 -> 357 MiB, 3.44x -> 2.43x the raw vectors). The cache kept + every embedding twice — a `Vec>` and a flattened copy for the + batched kernels, maintained in lock-step — so it now stores only the flat + buffer and indexes into it. Recall and query latency are unchanged. + **Breaking:** `MemoryCache::embeddings` is a `cache::Embeddings` rather than + a `Vec>` (indexing still yields a `&[f32]` row); `embeddings_flat` + is gone, replaced by `flat_embeddings()`; `rebuild_flat()` is a deprecated + no-op. Rows are now always exactly `dim` long — shorter ones are + zero-padded — which makes the ragged-row case that used to silently + misalign the flattened copy unrepresentable. +- `clawhdf5-bench`: `search_harness --footprint` reports live heap use per + stage, measured with a counting allocator (RSS cannot see a structure freed + into the allocator's own pool). + +### Testing +- The Python interop suites honour **`CLAWHDF5_PYTHON`**, and `ci-test.sh` + picks up a `.venv/bin/python` automatically. On a PEP 668 "externally + managed" system h5py cannot be installed into the system interpreter at all, + so every interop suite — the h5py writer round-trips, the facade, netCDF4 + and the reference files — was skipping silently. A silent skip here is + exactly how the v5 compound-datatype bug reached a release. + `CLAWHDF5_REQUIRE_INTEROP=1` still turns a skip into a failure. + ## v2.5.0 (2026-09-19) ### Upgrade Notes diff --git a/Cargo.toml b/Cargo.toml index 0623012..4b26e26 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,7 +21,7 @@ members = [ resolver = "2" [workspace.package] -version = "2.5.0" +version = "2.6.0" edition = "2024" license = "MIT" repository = "https://git.redclaw.dev/quantumclaw/clawhdf5" diff --git a/crates/clawhdf5-accel/Cargo.toml b/crates/clawhdf5-accel/Cargo.toml index d8e679d..5f82a0c 100644 --- a/crates/clawhdf5-accel/Cargo.toml +++ b/crates/clawhdf5-accel/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "clawhdf5-accel" -version = "2.5.0" +version = "2.6.0" edition = "2024" description = "SIMD-accelerated operations for rustyhdf5" license = "MIT" diff --git a/crates/clawhdf5-agent/Cargo.toml b/crates/clawhdf5-agent/Cargo.toml index 865d521..f531bb3 100644 --- a/crates/clawhdf5-agent/Cargo.toml +++ b/crates/clawhdf5-agent/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "clawhdf5-agent" -version = "2.5.0" +version = "2.6.0" edition = "2024" description = "HDF5-backed persistent memory store for on-device AI agents" license = "MIT" @@ -10,12 +10,12 @@ keywords = ["agent", "memory", "hdf5", "vector-search", "embedding"] categories = ["database", "science", "algorithms"] [dependencies] -clawhdf5-format = { path = "../clawhdf5-format", version = "2.5.0", features = ["parallel", "fast-checksum"] } -clawhdf5 = { path = "../clawhdf5", version = "2.5.0" } -clawhdf5-io = { path = "../clawhdf5-io", version = "2.5.0", features = ["mmap"] } -clawhdf5-accel = { path = "../clawhdf5-accel", version = "2.5.0" } -clawhdf5-ann = { path = "../clawhdf5-ann", version = "2.5.0", optional = true } -clawhdf5-gpu = { path = "../clawhdf5-gpu", version = "2.5.0", optional = true, default-features = false } +clawhdf5-format = { path = "../clawhdf5-format", version = "2.6.0", features = ["parallel", "fast-checksum"] } +clawhdf5 = { path = "../clawhdf5", version = "2.6.0" } +clawhdf5-io = { path = "../clawhdf5-io", version = "2.6.0", features = ["mmap"] } +clawhdf5-accel = { path = "../clawhdf5-accel", version = "2.6.0" } +clawhdf5-ann = { path = "../clawhdf5-ann", version = "2.6.0", optional = true } +clawhdf5-gpu = { path = "../clawhdf5-gpu", version = "2.6.0", optional = true, default-features = false } serde = { workspace = true } byteorder = "1" half = { workspace = true, optional = true } diff --git a/crates/clawhdf5-android/Cargo.toml b/crates/clawhdf5-android/Cargo.toml index bfe4780..0438a37 100644 --- a/crates/clawhdf5-android/Cargo.toml +++ b/crates/clawhdf5-android/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "clawhdf5-android" -version = "2.5.0" +version = "2.6.0" edition = "2024" description = "Android JNI bridge for edgehdf5-memory HDF5 backend" license = "MIT" diff --git a/crates/clawhdf5-ann/Cargo.toml b/crates/clawhdf5-ann/Cargo.toml index 49dab1e..0785b99 100644 --- a/crates/clawhdf5-ann/Cargo.toml +++ b/crates/clawhdf5-ann/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "clawhdf5-ann" -version = "2.5.0" +version = "2.6.0" edition = "2024" description = "HNSW approximate nearest neighbor index stored as HDF5" license = "MIT" @@ -10,9 +10,9 @@ keywords = ["hdf5", "ann", "hnsw", "nearest-neighbor"] categories = ["algorithms", "science"] [dependencies] -clawhdf5-format = { path = "../clawhdf5-format", version = "2.5.0" } -clawhdf5-io = { path = "../clawhdf5-io", version = "2.5.0" } -clawhdf5-accel = { path = "../clawhdf5-accel", version = "2.5.0" } +clawhdf5-format = { path = "../clawhdf5-format", version = "2.6.0" } +clawhdf5-io = { path = "../clawhdf5-io", version = "2.6.0" } +clawhdf5-accel = { path = "../clawhdf5-accel", version = "2.6.0" } rayon = { version = "1", optional = true } [features] diff --git a/crates/clawhdf5-bench/Cargo.toml b/crates/clawhdf5-bench/Cargo.toml index 1c8cb8c..3a14833 100644 --- a/crates/clawhdf5-bench/Cargo.toml +++ b/crates/clawhdf5-bench/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "clawhdf5-bench" -version = "2.5.0" +version = "2.6.0" edition = "2024" description = "Benchmark harnesses for clawhdf5-agent (Track 8)" license = "MIT" diff --git a/crates/clawhdf5-cli/Cargo.toml b/crates/clawhdf5-cli/Cargo.toml index 77f2825..5af9682 100644 --- a/crates/clawhdf5-cli/Cargo.toml +++ b/crates/clawhdf5-cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "clawhdf5-cli" -version = "2.5.0" +version = "2.6.0" edition = "2024" license = "MIT" description = "CLI for clawhdf5 agent memory — create, save, search, recall, stats" @@ -14,7 +14,7 @@ name = "clawhdf5" path = "src/main.rs" [dependencies] -clawhdf5-agent = { path = "../clawhdf5-agent", version = "2.5.0" } +clawhdf5-agent = { path = "../clawhdf5-agent", version = "2.6.0" } clap = { version = "4", features = ["derive", "env"] } serde_json = "1" serde = { workspace = true } diff --git a/crates/clawhdf5-derive/Cargo.toml b/crates/clawhdf5-derive/Cargo.toml index b0eb52a..67f458b 100644 --- a/crates/clawhdf5-derive/Cargo.toml +++ b/crates/clawhdf5-derive/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "clawhdf5-derive" -version = "2.5.0" +version = "2.6.0" edition = "2024" description = "Derive macros for rustyhdf5 HDF5 traits" license = "MIT" diff --git a/crates/clawhdf5-filters/Cargo.toml b/crates/clawhdf5-filters/Cargo.toml index 3f8a113..860bf78 100644 --- a/crates/clawhdf5-filters/Cargo.toml +++ b/crates/clawhdf5-filters/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "clawhdf5-filters" -version = "2.5.0" +version = "2.6.0" edition = "2024" description = "Filter and compression pipeline for clawhdf5" license = "MIT" diff --git a/crates/clawhdf5-format/Cargo.toml b/crates/clawhdf5-format/Cargo.toml index 2ee5748..801d653 100644 --- a/crates/clawhdf5-format/Cargo.toml +++ b/crates/clawhdf5-format/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "clawhdf5-format" -version = "2.5.0" +version = "2.6.0" edition = "2024" description = "Pure-Rust HDF5 binary format parsing and writing — no C dependencies" license = "MIT" @@ -25,7 +25,7 @@ pco = { version = "1.0", optional = true } [dev-dependencies] serde_json = "1" criterion = { workspace = true } -clawhdf5-derive = { path = "../clawhdf5-derive", version = "2.5.0" } +clawhdf5-derive = { path = "../clawhdf5-derive", version = "2.6.0" } [[bench]] name = "bench" diff --git a/crates/clawhdf5-gpu/Cargo.toml b/crates/clawhdf5-gpu/Cargo.toml index a05de3e..92df329 100644 --- a/crates/clawhdf5-gpu/Cargo.toml +++ b/crates/clawhdf5-gpu/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "clawhdf5-gpu" -version = "2.5.0" +version = "2.6.0" edition = "2024" description = "GPU-accelerated vector operations for rustyhdf5 using wgpu compute shaders" license = "MIT" diff --git a/crates/clawhdf5-io/Cargo.toml b/crates/clawhdf5-io/Cargo.toml index b5fd80c..e66c45d 100644 --- a/crates/clawhdf5-io/Cargo.toml +++ b/crates/clawhdf5-io/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "clawhdf5-io" -version = "2.5.0" +version = "2.6.0" edition = "2024" description = "I/O abstraction layer for rustyhdf5" license = "MIT" @@ -10,7 +10,7 @@ keywords = ["hdf5", "io", "science", "data"] categories = ["filesystem", "science"] [dependencies] -clawhdf5-format = { path = "../clawhdf5-format", version = "2.5.0" } +clawhdf5-format = { path = "../clawhdf5-format", version = "2.6.0" } memmap2 = { version = "0.9", optional = true } libc = { version = "0.2", optional = true } tokio = { version = "1", features = ["fs", "io-util"], optional = true } diff --git a/crates/clawhdf5-migrate/Cargo.toml b/crates/clawhdf5-migrate/Cargo.toml index ad48db6..4e2d4f6 100644 --- a/crates/clawhdf5-migrate/Cargo.toml +++ b/crates/clawhdf5-migrate/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "clawhdf5-migrate" -version = "2.5.0" +version = "2.6.0" edition = "2024" description = "CLI to migrate SQLite agent memory databases to HDF5 format" license = "MIT" @@ -14,9 +14,9 @@ name = "clawhdf5-migrate" path = "src/main.rs" [dependencies] -clawhdf5-agent = { path = "../clawhdf5-agent", version = "2.5.0" } -clawhdf5-format = { path = "../clawhdf5-format", version = "2.5.0" } -clawhdf5 = { path = "../clawhdf5", version = "2.5.0" } +clawhdf5-agent = { path = "../clawhdf5-agent", version = "2.6.0" } +clawhdf5-format = { path = "../clawhdf5-format", version = "2.6.0" } +clawhdf5 = { path = "../clawhdf5", version = "2.6.0" } rusqlite = { version = "0.31", features = ["bundled"] } clap = { version = "4", features = ["derive"] } half = { workspace = true } diff --git a/crates/clawhdf5-napi/Cargo.toml b/crates/clawhdf5-napi/Cargo.toml index 764a28c..c2be19e 100644 --- a/crates/clawhdf5-napi/Cargo.toml +++ b/crates/clawhdf5-napi/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "clawhdf5-napi" -version = "2.5.0" +version = "2.6.0" edition = "2024" description = "Node.js native addon (napi-rs) exposing clawhdf5-agent to TypeScript/JavaScript" license = "MIT" @@ -10,7 +10,7 @@ repository = "https://git.redclaw.dev/quantumclaw/clawhdf5" crate-type = ["cdylib"] [dependencies] -clawhdf5-agent = { path = "../clawhdf5-agent", version = "2.5.0" } +clawhdf5-agent = { path = "../clawhdf5-agent", version = "2.6.0" } napi = { version = "2", default-features = false, features = ["napi9"] } napi-derive = "2" diff --git a/crates/clawhdf5-netcdf4/Cargo.toml b/crates/clawhdf5-netcdf4/Cargo.toml index a88ad2a..b2a4d7a 100644 --- a/crates/clawhdf5-netcdf4/Cargo.toml +++ b/crates/clawhdf5-netcdf4/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "clawhdf5-netcdf4" -version = "2.5.0" +version = "2.6.0" edition = "2024" description = "NetCDF-4 read support built on rustyhdf5 — pure Rust, no C dependencies" license = "MIT" @@ -10,8 +10,8 @@ keywords = ["netcdf", "netcdf4", "hdf5", "science", "climate"] categories = ["parser-implementations", "science"] [dependencies] -clawhdf5 = { path = "../clawhdf5", version = "2.5.0" } -clawhdf5-format = { path = "../clawhdf5-format", version = "2.5.0" } +clawhdf5 = { path = "../clawhdf5", version = "2.6.0" } +clawhdf5-format = { path = "../clawhdf5-format", version = "2.6.0" } [dev-dependencies] tempfile = { workspace = true } diff --git a/crates/clawhdf5-py/Cargo.toml b/crates/clawhdf5-py/Cargo.toml index 774e12b..922353d 100644 --- a/crates/clawhdf5-py/Cargo.toml +++ b/crates/clawhdf5-py/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "clawhdf5-py" -version = "2.5.0" +version = "2.6.0" edition = "2024" description = "Python bindings for rustyhdf5 — a pure-Rust HDF5 library" license = "MIT" @@ -14,8 +14,8 @@ name = "clawhdf5" crate-type = ["cdylib", "rlib"] [dependencies] -clawhdf5_rs = { path = "../clawhdf5", version = "2.5.0", package = "clawhdf5" } -clawhdf5-format = { path = "../clawhdf5-format", version = "2.5.0" } +clawhdf5_rs = { path = "../clawhdf5", version = "2.6.0", package = "clawhdf5" } +clawhdf5-format = { path = "../clawhdf5-format", version = "2.6.0" } pyo3 = "0.29" numpy = "0.29" diff --git a/crates/clawhdf5-py/pyproject.toml b/crates/clawhdf5-py/pyproject.toml index ea4a41e..a2bed56 100644 --- a/crates/clawhdf5-py/pyproject.toml +++ b/crates/clawhdf5-py/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "maturin" [project] name = "rustyhdf5" -version = "2.5.0" +version = "2.6.0" description = "Python bindings for rustyhdf5 — a pure-Rust HDF5 library" requires-python = ">=3.8" license = { text = "MIT" } diff --git a/crates/clawhdf5/Cargo.toml b/crates/clawhdf5/Cargo.toml index a16511d..765d7ea 100644 --- a/crates/clawhdf5/Cargo.toml +++ b/crates/clawhdf5/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "clawhdf5" -version = "2.5.0" +version = "2.6.0" edition = "2024" description = "Pure-Rust HDF5 reader/writer — no C dependencies" license = "MIT" @@ -10,16 +10,16 @@ keywords = ["hdf5", "science", "data", "binary"] categories = ["parser-implementations", "science", "encoding"] [dependencies] -clawhdf5-format = { path = "../clawhdf5-format", version = "2.5.0" } -clawhdf5-io = { path = "../clawhdf5-io", version = "2.5.0" } +clawhdf5-format = { path = "../clawhdf5-format", version = "2.6.0" } +clawhdf5-io = { path = "../clawhdf5-io", version = "2.6.0" } rayon = { version = "1", optional = true } [dev-dependencies] tempfile = { workspace = true } criterion = { workspace = true } -clawhdf5-io = { path = "../clawhdf5-io", version = "2.5.0", features = ["mmap"] } -clawhdf5-format = { path = "../clawhdf5-format", version = "2.5.0", features = ["parallel", "fast-checksum"] } -clawhdf5-filters = { path = "../clawhdf5-filters", version = "2.5.0" } +clawhdf5-io = { path = "../clawhdf5-io", version = "2.6.0", features = ["mmap"] } +clawhdf5-format = { path = "../clawhdf5-format", version = "2.6.0", features = ["parallel", "fast-checksum"] } +clawhdf5-filters = { path = "../clawhdf5-filters", version = "2.6.0" } [[bench]] name = "mmap_bench" diff --git a/packages/clawhdf5-node/package.json b/packages/clawhdf5-node/package.json index a11acd9..f0acce2 100644 --- a/packages/clawhdf5-node/package.json +++ b/packages/clawhdf5-node/package.json @@ -1,6 +1,6 @@ { "name": "@redclaw/clawhdf5", - "version": "2.5.0", + "version": "2.6.0", "description": "Node.js bindings for clawhdf5 — HDF5-backed agent memory with hippocampal consolidation", "main": "index.js", "types": "index.d.ts",