From e9b37a9602c8911fcd78c332f65efd9fd1077e0d Mon Sep 17 00:00:00 2001 From: osobh Date: Wed, 23 Sep 2026 11:05:10 -0500 Subject: [PATCH 1/3] docs: bring the README up to date with the last five releases The README had fallen behind v2.3.0-v2.7.0, and parts of it were not true. Checked every claim against the code and BENCHMARKS.md: - Three of the six Quick Start snippets no longer compiled (Agent Memory, Consolidation, OpenClaw); all six now do. - Hybrid search was described as RRF throughout. The default has been weighted 0.4/0.6 fusion since v2.5.0; re-ranking and confidence rejection run only in the OpenClaw backend. - The `float16` feature does not halve embedding storage (the store always writes f32), `--features agent` enables nothing, "Source Isolation" is not wired in, and nothing backs "billion-scale" IVF-PQ. - "Cryptographically verifiable" overstated an unkeyed, session-scoped FNV-1a ledger; "Zero C dependencies" was false while zlib-ng was the default deflate backend. - Stale numbers: tests (1,650 -> 1,868), Rust badge (1.75 is below edition 2024's floor), 6.5 KB/record on disk (BENCHMARKS.md: 1.7 KB), consolidation and hybrid-search latency, and a feature-flag table broken by a paragraph pasted into it. - The file schema, module table and crate map now match the code. Adds a "What's new (v2.2 -> v2.7)" section for collaborators, leading with the silent Extensible Array read bug fixed in v2.7.0. Footer links point at git.redclaw.dev. CLAUDE.md: clawhdf5-migrate is the SQLite migration tool, and MemoryConfig::compression is off by default. Co-Authored-By: Claude Opus 5.5 (1M context) --- CLAUDE.md | 6 +- README.md | 403 +++++++++++++++++++++++++++++---------------- docs/QUICKSTART.md | 2 +- docs/USE_CASES.md | 2 +- 4 files changed, 268 insertions(+), 145 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 8a93702..6165035 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -19,7 +19,7 @@ Cargo workspace with 16 crates under `crates/` (plus `libaec-sys`, an internal F | `clawhdf5-agent` | Agent memory, session history, knowledge graph storage | | `clawhdf5-gpu` | GPU-accelerated I/O via wgpu (hand-written WGSL compute shaders) | | `clawhdf5-accel` | CPU SIMD acceleration path | -| `clawhdf5-migrate` | Schema migration engine | +| `clawhdf5-migrate` | SQLite → HDF5 agent-memory migration | | `clawhdf5-android` | Android JNI bindings | | `clawhdf5-cli` | Command-line interface | | `clawhdf5-napi` | Node.js native addon bindings | @@ -82,8 +82,8 @@ Cargo workspace with 16 crates under `crates/` (plus `libaec-sys`, an internal F `export` do). An unreadable WAL (torn header, bad magic) is quarantined to `.h5.wal.corrupt-` rather than blocking `open()`; a WAL with an unknown *newer* version still fails and is left untouched. -- `MemoryConfig::compression` uses deflate by default; enable the agent's - `zstd` feature to compress embeddings with Zstd instead (links libzstd). +- `MemoryConfig::compression` is off by default; when on, embeddings are + deflate-compressed, or Zstd with the agent's `zstd` feature (links libzstd). - `Dataset::verify_provenance()` (clawhdf5 facade, `provenance` feature, on by default) recomputes a dataset's SHA-256 and compares it against the `_provenance_sha256` attribute written automatically on save when diff --git a/README.md b/README.md index e235648..990b1c3 100644 --- a/README.md +++ b/README.md @@ -1,26 +1,85 @@ # ClawhDF5 -**The memory layer AI agents deserve. One file. Pure Rust. Zero C dependencies.** +**The memory layer AI agents deserve. One file. Pure Rust. No libhdf5.** [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE) -[![Rust](https://img.shields.io/badge/rust-1.75%2B-orange.svg)](https://www.rust-lang.org) -[![Tests](https://img.shields.io/badge/tests-1650%2B%20passing-brightgreen.svg)](#performance) -[![LongMemEval](https://img.shields.io/badge/LongMemEval%20oracle-Turn--Level%20Hit@5%2084%25%20BM25--only-blue.svg)](BENCHMARKS.md#longmemeval-results) -[![Footprint](https://img.shields.io/badge/footprint-6.5%20KB%2Frecord-lightgrey.svg)](BENCHMARKS.md#memory-footprint) +[![Rust](https://img.shields.io/badge/rust-1.94%2B-orange.svg)](https://www.rust-lang.org) +[![Tests](https://img.shields.io/badge/tests-1850%2B-brightgreen.svg)](#building) +[![LongMemEval](https://img.shields.io/badge/LongMemEval__s-Turn--Level%20Hit@5%2081.4%25%20hybrid-blue.svg)](BENCHMARKS.md#longmemeval-results) +[![Footprint](https://img.shields.io/badge/on--disk-1.7%20KB%2Frecord-lightgrey.svg)](BENCHMARKS.md#memory-footprint-1) -ClawHDF5 is a pure-Rust HDF5 implementation combined with a research-grade agent memory engine. It gives AI agents persistent, searchable, cryptographically verifiable memory — all stored in a single portable file. +ClawHDF5 is a pure-Rust HDF5 implementation combined with a research-grade agent memory engine. It gives AI agents persistent, searchable, integrity-checked memory — all stored in a single portable file. > **Two things live here:** -> - **A general-purpose, pure-Rust HDF5 library** — zero C dependencies, NetCDF-4 support, SIMD/GPU acceleration. See the **[Crate Map](#crate-map)** and **[BENCHMARKS.md](BENCHMARKS.md)** for the libhdf5 head-to-head numbers. +> - **A general-purpose, pure-Rust HDF5 library** — no libhdf5, NetCDF-4 support, SIMD/GPU acceleration. See the **[Crate Map](#crate-map)** and **[BENCHMARKS.md](BENCHMARKS.md)** for the libhdf5 head-to-head numbers. > - **An agent memory layer built on top of it** — vector search, knowledge graph, hippocampal-style consolidation, in `clawhdf5-agent`. -``` -cargo add clawhdf5 # core HDF5 read/write, no agent layer -cargo add clawhdf5-agent --features agent # + agent memory layer +The crates are not on crates.io yet, so depend on them from git: + +```toml +[dependencies] +clawhdf5 = { git = "https://git.redclaw.dev/quantumclaw/clawhdf5" } # core HDF5 read/write +clawhdf5-agent = { git = "https://git.redclaw.dev/quantumclaw/clawhdf5" } # + agent memory layer ``` +> **C dependencies, precisely:** the HDF5 format code is pure Rust and never +> links libhdf5. The default deflate backend is zlib-ng (`fast-deflate`), a C +> library built from source, so a default build needs `cmake` and a C +> compiler. Opt-in codecs (`zstd`, `szip`) and BLAS backends link C too. + > **New here?** Start with the **[Quickstart Guide](docs/QUICKSTART.md)** · See **[Use Cases](docs/USE_CASES.md)** · Read **[Benchmarks](BENCHMARKS.md)** +## What's new (v2.2 → v2.7, and unreleased) + +Five releases in September 2026. Details, including upgrade notes and every +breaking change, are in [CHANGELOG.md](CHANGELOG.md). + +**HDF5 correctness (read these if you read files with an earlier release)** +- **Extensible Array chunk indexes returned wrong data** past the 36th chunk — + any dataset with one unlimited dimension. Silent: plausible numbers from the + wrong chunks. Fixed in v2.7.0; re-read affected data. +- Fixed and Extensible Array checksums are now verified, so a corrupt chunk + index is `ChecksumMismatch` instead of wrong data (v2.7.0). +- Compound datatypes written with default libver bounds (plain + `h5py.File(path, 'w')`) were mis-parsed; HDF5 2.0 compound v5 and native + complex (class 11) types now parse (v2.2.0–v2.3.0). +- Committed datatypes, fill values, soft links and `H5T_STD_REF` references now + read correctly; external links and external raw data are explicit errors; + `attrs()` no longer silently drops attributes (v2.3.0–v2.5.0). +- Datasets indexed by a version-2 B-tree now read (v2.5.0). + +**Security and robustness** +- A crafted file could abort any reader via B-tree v2 recursion or explode it + via shared children; both are now fast errors (v2.7.0). +- Virtual-dataset source paths are confined to the file's directory; chunked + reads use overflow-checked sizes and fallible allocation, and the facade + writes files atomically (v2.3.0). +- Agent store: single-writer lock plus `open_read_only`; a crash between + checkpoint and WAL truncate no longer duplicates entries; unreadable WALs are + quarantined instead of blocking `open()` (v2.3.0). + +**Search quality and speed** +- HNSW neighbour selection now uses the paper's diversity heuristic: recall@10 + at 100K went from 0.31 to 0.98 (v2.4.0). +- `hybrid_search` is 79–190× faster than v2.3.0 (p50 0.07 ms at 1K, 4.65 ms at + 100K). It no longer rebuilds BM25 or rewrites the store per query, and the + HNSW graph is persisted (v2.4.0). +- Default fusion weights are now the measured 0.4 / 0.6 (v2.5.0). Re-ranking had + been discarding the retrieval score, costing the OpenClaw backend 40.6pp of + Hit@1; fixed in v2.6.0. +- Selection reads decode only the chunks they touch (a 64×64 window: 105 ms to + 0.39 ms), and full reads are 1.2–1.9× faster (v2.5.0). + +**Memory** +- A loaded store holds ~30% less (embeddings stored once, v2.6.0), and the + int8 HNSW index, **on by default for new stores** (unreleased), brings a + 100K × 384 store to 1.74× the raw vectors. At equal recall it is also faster + than `f32`: 1.63× QPS on AVX2, 1.18× on a Raspberry Pi 5 (NEON `SDOT`). + +**Tooling** +- CI now runs the h5py/netCDF4 interop suites for real (they had been skipping + silently) and runs an aarch64 job for the NEON kernels. + --- ## Why ClawhDF5? @@ -35,7 +94,7 @@ Every AI agent needs memory. Today that means scattered Markdown files, SQLite d | Memory consolidation | Manual pruning | Hippocampal-inspired automatic tiers | | Temporal queries | Custom code | Native temporal index (716ns) | | Multi-modal | Multiple stores | Unified cross-modal search | -| Security | Hope for the best | Provenance tracking + anomaly detection | +| Integrity | Hope for the best | Chained-CRC WAL, checksummed chunk indexes, write-anomaly alerts, opt-in SHA-256 dataset provenance | | Portability | Config + DB + files | **One `.h5` file. Copy it anywhere.** | --- @@ -60,6 +119,20 @@ Figures below are from an independent reproduction run on a second machine (AMD ### Vector Search +**HNSW (the default backend for `hybrid_search`)** — `search_harness`, clustered +384-dim data, M = 16, ef_construction = 64, recall measured against an exact scan. +See [BENCHMARKS.md § Search harness](BENCHMARKS.md#search-harness-baseline-v230) +and [§ Quantising the index copy](BENCHMARKS.md#quantising-the-index-copy-quantized_index): + +| N = 100K, ef = 64 | recall@10 | QPS | build | +|---|---:|---:|---:| +| `f32` index | 0.9945 | 13 399 | 3.2 s | +| `i8` index + exact re-score (**default for new stores**) | 0.9940 | **21 848** | **1.8 s** | + +Before the v2.4.0 neighbour-selection fix, recall@10 at 100K was 0.31. + +**Brute-force and IVF paths** (Criterion, i7-12650H): + | Scale | Flat | IVF (nprobe=10) | IVF-PQ | vs MemX¹ | |-------|------|-----------------|--------|----------| | 1K | **54 µs** | — | — | — | @@ -75,7 +148,7 @@ Figures below are from an independent reproduction run on a second machine (AMD | Operation | Latency | Scale | |-----------|---------|-------| -| Hybrid search (RRF) | **222 µs** | 1K records | +| Hybrid search (`HDF5Memory::hybrid_search`, p50) | **70 µs** / 0.49 ms / 4.65 ms | 1K / 10K / 100K records | | BM25 keyword search | **67 µs** | 1K records | | Knowledge graph BFS | **24 µs** | 1K entities | | Spreading activation | **17 µs** | 100 entities | @@ -115,13 +188,17 @@ declaration: Hybrid is the strongest configuration, which is what running two retrieval stages is for. The weights matter more than the stages: a sweep of `vector_weight` from -0.0 to 1.0 found the long-standing `0.7/0.3` default is **strictly dominated** by -`0.4/0.6` — better on Hit@1, Hit@5, Hit@10 and MRR at both granularities. Use -`0.4/0.6`, or `0.3/0.7` if rank-1 precision matters most. See -[BENCHMARKS.md § Weight sweep](BENCHMARKS.md#longmemeval-results). +0.0 to 1.0 found the old `0.7/0.3` default is **strictly dominated** by +`0.4/0.6` — better on Hit@1, Hit@5, Hit@10 and MRR at both granularities. Since +v2.5.0 `0.4/0.6` is the default (`hybrid::DEFAULT_FUSION`, used by +`unified_search`, `hybrid_search_with` and the OpenClaw backend); callers that +pass weights to `hybrid_search` explicitly choose their own. Use `0.3/0.7` if +rank-1 precision matters most. Reciprocal rank fusion is selectable +(`hybrid::Fusion::Rrf`) but measured worse than the weighted sum. See +[BENCHMARKS.md § Weight sweep](BENCHMARKS.md#weight-sweep--full-haystack-n500). -Vector embeddings require `--features embeddings`; without it the vector stage is -inert and only the BM25 row is produced, which is what every previously published +The benchmark's vector stage requires `clawhdf5-bench`'s `embeddings` feature +(real MiniLM embeddings); without it the vector stage is inert and only the BM25 row is produced, which is what every previously published number here measured. On the easier `longmemeval_oracle` variant (evidence sessions only) the same @@ -146,19 +223,37 @@ retrieval recall reported as QA accuracy typically overstates by 20–30 points. ### Memory Footprint -| Records | File Size | Bytes/Record | With Compression | -|---------|-----------|--------------|------------------| -| 1K | ~6.5 MB | ~6.5 KB | ~2.1 MB (3.1x) | -| 10K | ~65 MB | ~6.5 KB | ~21 MB (3.1x) | -| 100K | ~645 MB | ~6.5 KB | ~208 MB (3.1x) | +**On disk** — 384-dim embeddings, 200-char text +([BENCHMARKS.md § Memory Footprint](BENCHMARKS.md#memory-footprint-1)): + +| Records | File Size | Bytes/Record | Gzip-6 compressed | +|---------|-----------|--------------|-------------------| +| 1K | 1.7 MB | 1.8 KB | 277 KB (6.1x) | +| 10K | 17.0 MB | 1.7 KB | 2.7 MB (6.2x) | +| 100K | 169.8 MB | 1.7 KB | 26.9 MB (6.2x) | + +**In memory** — a store reopened from disk, 384-dim `f32`, measured with a +counting allocator ([BENCHMARKS.md § Memory footprint](BENCHMARKS.md#memory-footprint)): + +| Records | Raw vectors | Reopened, `f32` index | Reopened, `i8` index (default) | +|---------|-------------|-----------------------|--------------------------------| +| 1K | 1 MiB | 4 MiB (2.40x) | 2 MiB (1.64x) | +| 10K | 15 MiB | 44 MiB (3.03x) | 27 MiB (1.81x) | +| 100K | 146 MiB | 399 MiB (2.72x) | **256 MiB (1.74x)** | + +Down from 505 MiB (3.44x) at 100K before v2.6.0, when the cache held every +embedding twice. ### Consolidation Efficiency +1,000 records (10 signal + 990 noise), `working_capacity = 100` +([BENCHMARKS.md § Consolidation Efficiency](BENCHMARKS.md#consolidation-efficiency)): + | Metric | Before | After | Delta | |--------|--------|-------|-------| -| Records in store | 1,000 | ~110 | −89% | -| Hit@1 recall | ~60% | ~90% | +30% | -| Search latency | ~2.8 ms | ~0.3 ms | **9x faster** | +| Records in store | 1,000 | 100 | −90% | +| Hit@1 recall (signal records) | 100% | 100% | no loss | +| Search latency | 2.75 ms | 0.31 ms | **8.8x faster** | **Full benchmark details: [BENCHMARKS.md](BENCHMARKS.md)** @@ -166,74 +261,71 @@ retrieval recall reported as QA accuracy typically overstates by 20–30 points. ## Agent Memory Architecture -ClawhDF5's agent memory engine implements research from 15+ recent papers on agentic memory systems. It's not a toy — it's the real thing. +ClawhDF5's agent memory engine draws on 15+ recent papers on agentic memory systems (see [Research Foundation](#research-foundation)). ``` - ┌─────────────────┐ - │ Agent Query │ - └────────┬────────┘ - │ - ┌────────────▼────────────┐ - │ Hybrid Retrieval │ - │ Vector + BM25 + RRF │ - └────────────┬────────────┘ - │ - ┌──────────────────▼──────────────────┐ - │ Multi-Factor Re-Ranking │ - │ temporal · authority · activation │ - └──────────────────┬──────────────────┘ - │ - ┌────────────▼────────────┐ - │ Confidence Rejection │ - │ (suppress bad matches) │ - └────────────┬────────────┘ - │ - ┌────────────────────────▼────────────────────────┐ - │ Memory Store (HDF5) │ - │ │ - │ ┌───────────┐ ┌───────────┐ ┌───────────────┐ │ - │ │ Working │→│ Episodic │→│ Semantic │ │ - │ │ (bounded) │ │ (bounded) │ │ (long-term) │ │ - │ └───────────┘ └───────────┘ └───────────────┘ │ - │ │ - │ ┌──────────┐ ┌──────────┐ ┌────────────────┐ │ - │ │Knowledge │ │Temporal │ │ Multi-Modal │ │ - │ │ Graph │ │ Index │ │ Embeddings │ │ - │ └──────────┘ └──────────┘ └────────────────┘ │ - │ │ - │ ┌──────────┐ ┌──────────┐ ┌────────────────┐ │ - │ │Provenance│ │ Anomaly │ │ Source │ │ - │ │ Tracking │ │Detection │ │ Isolation │ │ - │ └──────────┘ └──────────┘ └────────────────┘ │ - └─────────────────────────────────────────────────┘ - │ - ┌────────┴────────┐ - │ agent_memory.h5 │ - │ single file │ - └─────────────────┘ + ┌─────────────────┐ + │ Agent Query │ + └────────┬────────┘ + │ + ┌─────────────────▼──────────────────┐ + │ HDF5Memory::hybrid_search │ + │ HNSW vector + BM25 keyword │ + │ weighted fusion (0.4 / 0.6) │ + │ × √(Hebbian activation) │ + └─────────────────┬──────────────────┘ + │ OpenClaw backend adds: + ┌─────────────────▼──────────────────┐ + │ Multi-factor re-ranking │ + │ relevance · recency · authority · │ + │ activation │ + ├────────────────────────────────────┤ + │ Confidence rejection │ + │ (suppress bad matches) │ + └─────────────────┬──────────────────┘ + │ + ┌────────────────────────────▼────────────────────────────┐ + │ In memory │ + │ cache (flat f32 embeddings) · BM25 index · HNSW index │ + │ provenance ledger + anomaly alerts (session-scoped) │ + └────────────────────────────┬────────────────────────────┘ + │ WAL append; checkpoint + ┌────────────────────────────▼────────────────────────────┐ + │ agent_memory.h5 /meta · /memory · /sessions · │ + │ /knowledge_graph │ + │ agent_memory.h5.wal chained-CRC write-ahead log │ + │ agent_memory.h5.ann HNSW graph (derived, rebuildable) │ + │ agent_memory.h5.lock single-writer lock │ + └─────────────────────────────────────────────────────────┘ ``` +Consolidation tiers (Working → Episodic → Semantic), the knowledge-graph +algorithms, temporal and multi-modal indexes are library components you drive +directly; the store persists the records, sessions and graph they work over. + ### Module Overview | Module | What It Does | |--------|-------------| -| **`knowledge`** | Entity/relation graph with BFS traversal, spreading activation, fuzzy entity resolution | -| **`consolidation`** | Three-tier memory (Working → Episodic → Semantic) with importance scoring and time-decay | -| **`hybrid`** | Vector + BM25 fusion with Reciprocal Rank Fusion (RRF, k=60). The vector stage uses the HNSW index by default (`hnsw` feature, on by default); disable with `--no-default-features --features float16` for an exact linear scan | -| **`reranker`** | Multi-factor re-ranking: temporal recency, source authority, activation weight | -| **`confidence`** | Low-confidence rejection — suppresses spurious recalls when nothing matches | +| **`knowledge`** | Entity/relation graph with BFS traversal, spreading activation, fuzzy (Levenshtein) entity resolution | +| **`consolidation`** | Three-tier memory (Working → Episodic → Semantic) with importance scoring, novelty, and time-decay | +| **`hybrid`** | Vector + BM25 fusion. Default is a min-max-normalised weighted sum, vector 0.4 / keyword 0.6 (`hybrid::DEFAULT_FUSION`, tuned on LongMemEval); RRF is available via `Fusion::Rrf` / `hybrid_search_with`. The vector stage uses the HNSW index by default (`hnsw` feature); disable with `--no-default-features --features float16` for an exact linear scan | +| **`reranker`** | Multi-factor re-ranking: retrieval relevance (leads, weight 1.0), temporal recency, source authority, activation weight. Used by the OpenClaw backend | +| **`confidence`** | Low-confidence rejection — suppresses spurious recalls when nothing matches (OpenClaw backend) | | **`temporal`** | Sorted timestamp index, session DAG, entity timeline, temporal query hints | | **`multimodal`** | Cross-modal search across text/image/audio/video embeddings | -| **`provenance`** | Source attribution, FNV-1a content hashing, integrity verification | -| **`anomaly`** | Write rate limiting, 15 injection pattern detectors, source distribution analysis | +| **`provenance`** | Source attribution and an unkeyed FNV-1a content hash per record, held in memory for the session, for detecting accidental corruption (not tamper-proof) | +| **`anomaly`** | Write rate limiting, 15 injection-pattern detectors, source-distribution analysis. Alerts never block a save; drain them with `take_anomaly_alerts` | | **`openclaw`** | OpenClaw integration: MemoryBackend trait, Markdown ↔ HDF5 conversion | | **`vector_search`** | Flat cosine, pre-normed, SIMD, BLAS, GPU, parallel search paths | -| **`ivf` / `pq`** | IVF-PQ approximate nearest neighbor for billion-scale search | -| **`bm25`** | BM25 keyword index with TF-IDF scoring | +| **`ivf` / `pq`** | Standalone IVF and IVF-PQ indexes (benchmarked to 100K vectors); not used by `HDF5Memory`, whose ANN index is HNSW | +| **`bm25`** | Incremental Okapi BM25 inverted index, kept for the life of the store; optional stemming | +| **`query_expand`** | Synonym / acronym / temporal query expansion | | **`entity_extract`** | Rule-based entity extraction from text chunks into the knowledge graph | -| **`wal`** | Write-ahead log for crash-safe persistence; each entry is CRC32-checked on replay, so a corrupted entry stops replay there instead of loading bad data | +| **`wal`** | Write-ahead log (v4) with a chained CRC32 per entry, so a corrupted, reordered, duplicated or spliced entry stops replay; checkpoints record a WAL mark so nothing is applied twice. Appends are not fsynced | | **`memory_strategy`** | Pluggable strategies: save-every, semantic-shift, user-correction detection | | **`decision_gate`** | Sub-microsecond trivial/substantive classification | +| **`ephemeral`** | In-memory TTL/LFU working tier | | **`async_memory`** | Tokio-based async wrapper over the memory store (`async` feature) | --- @@ -265,7 +357,7 @@ assert_eq!(values, vec![22.5, 23.1, 21.8]); use clawhdf5_agent::{HDF5Memory, MemoryConfig, MemoryEntry, AgentMemory}; // Create memory store -let config = MemoryConfig::new("agent.h5", "my-agent", 384); +let config = MemoryConfig::new("agent.h5".into(), "my-agent", 384); let mut memory = HDF5Memory::create(config)?; // Save a memory @@ -278,8 +370,8 @@ memory.save(MemoryEntry { tags: "preference".into(), })?; -// Search -let results = memory.search(&query_embedding, 5)?; +// Hybrid search: vector + BM25, weighted 0.4 / 0.6 (the measured default) +let results = memory.hybrid_search(&query_embedding, "user preferences", 0.4, 0.6, 5); for result in results { println!("[{:.3}] {}", result.score, result.chunk); } @@ -309,8 +401,8 @@ let neighbors = kg.bfs_neighbors(alice, 2); // 2-hop neighborhood let activated = kg.spreading_activation(&[alice], 0.5, 0.01, 5); // Entity resolution — fuzzy matching -let resolved = kg.resolve_or_create("alice", "person", -1, 2); -// Returns existing Alice entity (Levenshtein distance ≤ 2) +let (id, created) = kg.resolve_or_create("alice", "person", -1, 2); +// id == alice, created == false: matched the existing entity (Levenshtein distance ≤ 2) ``` ### Memory Consolidation @@ -321,15 +413,19 @@ use clawhdf5_agent::consolidation::*; let config = ConsolidationConfig::default(); let mut engine = ConsolidationEngine::new(config); -// Add memories — automatically scored for importance -engine.add_memory("User prefers dark mode", vec![0.1, 0.2, ...], MemorySource::User); -engine.add_memory("ok", vec![0.0, 0.0, ...], MemorySource::System); +let now = 1_700_000_000.0; // seconds since the epoch + +// Add memories — automatically scored for importance. +// Elevated sources (System, …) go through a separate, explicit API. +let id = engine.add_memory("User prefers dark mode".into(), vec![0.1, 0.2, ...], UntrustedSource::User, now); +engine.add_trusted_memory("ok".into(), vec![0.0, 0.0, ...], TrustedSource::System, now); // Access a memory (reactivates it) -engine.access_memory(0); +engine.access_memory(id, now); // Run consolidation cycle -let stats = engine.consolidate(); +engine.consolidate(now); +let stats = engine.get_stats(); // Working memories promote to Episodic (if important enough) // Episodic memories promote to Semantic (if accessed enough) // Low-decay memories get evicted when tiers are full @@ -357,13 +453,13 @@ let recent = index.latest(10); use clawhdf5_agent::openclaw::*; // Create backend -let mut backend = ClawhdfBackend::create("memory.h5", "agent-1", 384)?; +let mut backend = ClawhdfBackend::create(std::path::Path::new("memory.h5"), 384)?; // Ingest existing Markdown memory files let md = std::fs::read_to_string("MEMORY.md")?; let count = backend.ingest_markdown("MEMORY.md", &md)?; -// Search (uses full pipeline: RRF → re-rank → confidence filter) +// Search (full pipeline: weighted vector + BM25 fusion → re-rank → confidence filter) let results = backend.search("user preferences", &query_embedding, 5); // Export back to Markdown @@ -375,22 +471,23 @@ let exported = backend.export_markdown("MEMORY.md")?; ## Crate Map ``` -clawhdf5 workspace (16 crates, ~92K lines of Rust; plus libaec-sys, an - internal FFI bindings crate for the optional szip feature) +clawhdf5 workspace (16 crates, ~86K lines of Rust in src/, ~104K with tests + and benches; plus libaec-sys, an internal FFI bindings + crate for the optional szip feature) │ ├── Core HDF5 -│ ├── clawhdf5-format — Binary parser/writer (no_std), shared type definitions -│ ├── clawhdf5-io — I/O abstraction (buffered, mmap, async) +│ ├── clawhdf5-format — Binary parser/writer (no_std-capable), shared type definitions +│ ├── clawhdf5-io — I/O abstraction (file/memory readers; optional mmap, async, HSDS, MPI) │ ├── clawhdf5-filters — Fast deflate path (zlib-ng); lz4/zstd/pcodec/szip filters live in clawhdf5-format │ ├── clawhdf5-derive — Proc macros │ ├── clawhdf5 — High-level API │ ├── clawhdf5-netcdf4 — NetCDF-4 support -│ ├── clawhdf5-accel — SIMD (NEON, AVX2, AVX-512) +│ ├── clawhdf5-accel — SIMD (AVX2, NEON incl. SDOT int8; AVX-512 behind `avx512`) │ └── clawhdf5-gpu — GPU compute (wgpu, hand-written WGSL compute shaders) │ ├── Agent Memory -│ ├── clawhdf5-agent — Memory engine (20.9K lines, 32 modules; WAL is CRC32-checked per entry) -│ ├── clawhdf5-ann — HNSW approximate nearest neighbor (default backend; optional `parallel` feature) +│ ├── clawhdf5-agent — Memory engine (24.7K lines, 32 modules; chained-CRC WAL) +│ ├── clawhdf5-ann — HNSW approximate nearest neighbor (default backend; f32 or int8 storage; `parallel` build) │ ├── clawhdf5-migrate — SQLite → HDF5 migration │ ├── clawhdf5-android — Android JNI bridge │ └── clawhdf5-cli — CLI tool @@ -411,10 +508,10 @@ ClawhDF5's agent memory design draws from 15+ recent papers: | Paper | Key Insight | ClawhDF5 Module | |-------|-------------|-----------------| -| **MemX** (2026) | RRF + multi-factor re-ranking | `hybrid`, `reranker` | -| **Graph-Native Cognitive Memory** (2026) | Graph-structured belief revision | `knowledge` | +| **MemX** (2026) | Hybrid fusion + multi-factor re-ranking | `hybrid`, `reranker` | +| **Graph-Native Cognitive Memory** (2026) | Graph-structured memory (weighted, timestamped relations; entity timelines) | `knowledge`, `temporal` | | **CraniMem** (2026) | Bounded hippocampal memory | `consolidation` | -| **D-MEM** (2026) | Reward prediction error gating | `consolidation` | +| **D-MEM** (2026) | Surprise-gated storage (implemented as a novelty score) | `consolidation` | | **SYNAPSE** (2025) | Spreading activation for recall | `knowledge` | | **RAGdb** (2025) | Zero-dependency edge RAG | Architecture | | **MemoryGraft** (2025) | Memory poisoning attacks | `anomaly`, `provenance` | @@ -429,29 +526,35 @@ ClawhDF5's agent memory design draws from 15+ recent papers: | Flag | Default | Description | |------|---------|-------------| -| `agent` | no | Full agent memory layer | -| `float16` | **yes** | Half-precision embedding storage (2× compression) | +| `float16` | **yes** | Half-precision cosine kernel (`cosine_similarity_f16`). The store itself always writes `f32` embeddings; `MemoryConfig::float16` is recorded in `/meta` but not yet applied | | `hnsw` | **yes** | HNSW approximate vector index for `hybrid_search` (via `clawhdf5-ann`); disable for an exact linear scan | +| `parallel` | **yes** | Parallel HNSW bulk build (same graph, ~3× faster on 16 cores) and Rayon brute-force search strategies | +| `zstd` | no | Compress embeddings with Zstd instead of deflate when `MemoryConfig::compression` is on (links libzstd) | +| `fast-math` | no | BLAS matrix-vector multiply | +| `accelerate` | no | Apple Accelerate / AMX (macOS) | +| `openblas` | no | OpenBLAS (Linux) | +| `gpu` | no | GPU search via wgpu | +| `async` | no | Tokio async with background flush | +| `agent` | no | Reserved; currently enables nothing (the agent layer is always built) | + +To opt out of the parallel build: `--no-default-features --features float16,hnsw`. +For an exact linear cosine scan instead of HNSW: `--no-default-features --features float16`. `MemoryConfig::hnsw_m`, `hnsw_ef_construction` and `hnsw_ef_search` tune the vector index (16 / 64 / scale-with-`k` by default) and are stored with the file. `MemoryConfig::quantized_index` (**on by default** for new stores) holds the -HNSW index's own -copy of the embeddings as `i8`, roughly halving a loaded store's memory -(2.72x -> 1.74x the raw vectors at 100k x 384). Quantised distances are -approximate, so the query path re-scores the candidate pool against the exact -embeddings the store already holds, which keeps recall at the `f32` index's -level. It is also **faster**: 1.63x the queries per second at equal recall on -x86-64 (AVX2) and 1.18x on a Raspberry Pi 5 (NEON `SDOT`), with index builds -1.8x and 2.3x faster respectively. See `BENCHMARKS.md`, "Quantising the index copy". -| `parallel` | no | Rayon parallel search | -| `fast-math` | no | BLAS matrix-vector multiply | -| `accelerate` | no | Apple Accelerate / AMX (macOS) | -| `openblas` | no | OpenBLAS (Linux) | -| `gpu` | no | GPU search via wgpu | -| `async` | no | Tokio async with background flush | +HNSW index's own copy of the embeddings as `i8`, roughly halving a loaded +store's memory (2.72x -> 1.74x the raw vectors at 100k x 384). Quantised +distances are approximate, so the query path re-scores the candidate pool +against the exact embeddings the store already holds, which keeps recall at the +`f32` index's level. It is also **faster**: 1.63x the queries per second at +equal recall on x86-64 (AVX2) and 1.18x on a Raspberry Pi 5 (NEON `SDOT`), with +index builds 1.8x and 2.3x faster respectively. Stores created before the +setting existed keep their `f32` index; opt out for new stores with +`quantized_index = false` or `clawhdf5-cli create --f32-index`. See +[BENCHMARKS.md § Quantising the index copy](BENCHMARKS.md#quantising-the-index-copy-quantized_index). ### `clawhdf5-format` @@ -461,8 +564,8 @@ x86-64 (AVX2) and 1.18x on a Raspberry Pi 5 (NEON `SDOT`), with index builds | `deflate` | yes | Deflate compression | | `checksum` | yes | Jenkins lookup3 verification | | `provenance` | yes | SHA-256 provenance attributes | -| `fast-deflate` | **yes** | zlib-ng backend for faster deflate | -| `system-zlib-decompress` | **yes** | Use the system zlib for decompression where available | +| `fast-deflate` | **yes** | zlib-ng backend for faster deflate (C; needs `cmake`) | +| `system-zlib-decompress` | **yes** | Use Apple's system libz for decompression (macOS only; no effect elsewhere) | | `parallel` | no | Parallel chunk encoding + compression (rayon) | | `fast-checksum` | no | crc32fast-accelerated checksums | | `lz4` | no | LZ4 block compression filter (id 32004) | @@ -470,17 +573,21 @@ x86-64 (AVX2) and 1.18x on a Raspberry Pi 5 (NEON `SDOT`), with index builds | `pcodec` | no | Pcodec lossless numerical codec (id 32023, via `pco` crate) | | `system-zlib` / `zlib-rs` | no | Alternative zlib backends for deflate | | `blake3_hash` | no | BLAKE3 content hashing for provenance | +| `szip` | no | SZIP filter (id 4) via libaec (C, through the internal `libaec-sys` crate) | ### `clawhdf5-ann` | Flag | Default | Description | |------|---------|-------------| -| `parallel` | no | Rayon-parallel neighbor-distance computation during HNSW graph pruning | +| `parallel` | no | Batched bulk build runs neighbour planning and back-link pruning on a Rayon pool; the graph is identical with or without it (enabled by `clawhdf5-agent`'s default `parallel`) | ### `clawhdf5-io` | Flag | Default | Description | |------|---------|-------------| +| `mmap` | no | Memory-mapped reads (`memmap2`) | +| `async` | no | Tokio-based async I/O | +| `hsds` | no | HSDS (HDF REST service) client | | `mpi-io` | no | MPI-backed I/O via the `mpi` crate | > **Parallel I/O (MPI) limitation:** `mpi-io`'s read path is a root-rank read @@ -494,17 +601,17 @@ x86-64 (AVX2) and 1.18x on a Raspberry Pi 5 (NEON `SDOT`), with index builds ## Building ```bash -# Default +# Default (needs cmake + a C compiler for zlib-ng) cargo build --workspace # Agent memory with all accelerations (Linux) -cargo build -p clawhdf5-agent --features "agent,float16,parallel,fast-math" +cargo build -p clawhdf5-agent --features fast-math # Agent memory with Apple Accelerate (macOS) -cargo build -p clawhdf5-agent --features "agent,float16,accelerate,parallel,gpu" +cargo build -p clawhdf5-agent --features "accelerate,gpu" # Tests -cargo test --workspace # all 1,650+ tests +cargo test --workspace # all 1,850+ tests cargo test -p clawhdf5-agent # agent memory tests scripts/ci-test.sh # what CI runs: fmt, clippy matrix, tests, # h5py/netCDF4 interop, no_std @@ -526,25 +633,41 @@ cargo bench -p clawhdf5-bench # h5bench-equivalent I/O suite ``` agent_memory.h5 -├── /meta -│ ├── schema_version: "1.0" -│ ├── agent_id, embedder, embedding_dim -│ └── created_at +├── /meta (attributes) +│ ├── schema_version: "1.0", edgehdf5_version +│ ├── agent_id, embedder, embedding_dim, chunk_size, overlap, created_at +│ ├── float16, compression, compression_level, compact_threshold, +│ │ hebbian_boost, decay_factor, wal_enabled, wal_max_entries +│ ├── quantized_index, hnsw_m, hnsw_ef_construction, hnsw_ef_search +│ ├── wal_applied_len, wal_applied_crc (WAL mark of the last checkpoint) +│ └── ann_generation (ties the .ann sidecar to this checkpoint) ├── /memory -│ ├── chunks: string[N] -│ ├── embeddings: f32[N × D] (or f16 with float16 flag) -│ ├── tombstones: u8[N] -│ └── norms: f32[N] (pre-computed L2) +│ ├── chunks: string[N] +│ ├── embeddings: f32[N × D] (chunked; deflate, or Zstd with the +│ │ `zstd` feature, when compression is on) +│ ├── source_channel: string[N] +│ ├── timestamps: f64[N] +│ ├── session_ids: string[N] +│ ├── tags: string[N] +│ ├── tombstones: u8[N] +│ ├── norms: f32[N] (pre-computed L2) +│ └── activation_weights: f32[N] (Hebbian) ├── /sessions -│ ├── ids: string[S] -│ └── summaries: string[S] +│ ├── ids, channels, summaries: string[S] +│ ├── start_idxs, end_idxs: i64[S] +│ └── timestamps: f64[S] └── /knowledge_graph - ├── entity_names: string[E] - ├── relation_srcs: i64[R] - ├── relation_tgts: i64[R] - └── relation_types: string[R] + ├── entity_ids, entity_emb_idxs: i64[E]; entity_names, entity_types: string[E] + ├── relation_srcs, relation_tgts: i64[R]; relation_types: string[R] + ├── relation_weights: f32[R]; relation_ts: f64[R] + └── alias_strings: string[A]; alias_entity_ids: i64[A] (when aliases exist) ``` +Alongside the store: `.h5.wal` (write-ahead log), `.h5.ann` +(HNSW graph; derived, safe to delete) and `.h5.lock` (single-writer +lock). A second writer gets `MemoryError::Locked`; use +`HDF5Memory::open_read_only` for a lock-free point-in-time view. + --- ## Migration @@ -599,6 +722,6 @@ MIT ---

- Built by RedClaw Systems
- ~92,000 lines of Rust. Zero C dependencies. One file to remember everything. + Built by RedClaw Systems
+ ~86,000 lines of Rust. No libhdf5. One file to remember everything.

diff --git a/docs/QUICKSTART.md b/docs/QUICKSTART.md index c751768..0635ce0 100644 --- a/docs/QUICKSTART.md +++ b/docs/QUICKSTART.md @@ -567,4 +567,4 @@ let final_results = confidence::reject_low_confidence( --- -

Built by RedClaw Systems

+

Built by RedClaw Systems

diff --git a/docs/USE_CASES.md b/docs/USE_CASES.md index 1ac85a9..4d0cb4a 100644 --- a/docs/USE_CASES.md +++ b/docs/USE_CASES.md @@ -232,4 +232,4 @@ clawhdf5-agent = { version = "2.0", features = ["agent", "float16", "accelerate" --- -

Built by RedClaw Systems

+

Built by RedClaw Systems

-- 2.54.0 From 735db117a7112039e1855a0e337f070404362f03 Mon Sep 17 00:00:00 2001 From: osobh Date: Wed, 23 Sep 2026 11:05:22 -0500 Subject: [PATCH 2/3] build: pure-Rust zlib-rs as the default deflate backend The core crates (clawhdf5, -agent, -format, -io, -filters, -ann, -accel, -netcdf4, -cli) now build no C by default: deflate defaults to zlib-rs, a pure-Rust port of zlib-ng, and zlib-ng becomes the opt-in `fast-deflate`, which overrides zlib-rs wherever it is enabled. A default build no longer needs cmake or a C compiler. Measured on tank, both builds run alternately, three rounds, medians: zlib-rs is within 6% of zlib-ng on every HDF5 read and write (512x512 deflate-6 chunked write 1.458 vs 1.484 ms; 64 MB compressed read 64.4 vs 65.2 ms), and compressed output is byte-identical. Details in BENCHMARKS.md, "Deflate backend". Getting there took two fixes the first measurement exposed: - zlib-rs needs `std` to detect SIMD at runtime. flate2 enables it via its default `runtime_detection`, which `default-features = false` had switched off, leaving zlib-rs 3.5x slower on inflate. The `zlib-rs` features now enable it. - Both deflate paths streamed through flate2's 32 KiB read/write wrappers. They now hand the codec the whole chunk in one call, into a buffer sized up front (~5% on chunked writes). This also fixes a silent short read: the streaming reader returned a truncated stream's bytes without an error; a truncated chunk is now DecompressionError. In clawhdf5-filters, output longer than the stated size is now an error rather than silently cut off. CI: ci-test.sh lints and tests the zlib-ng path, and fails if a C-building crate (*-sys, cc, cmake) enters a core crate's default dependency tree. The arm64 job no longer installs cmake. Co-Authored-By: Claude Opus 5.5 (1M context) --- .gitea/workflows/ci.yml | 18 +- BENCHMARKS.md | 63 +++++++ CHANGELOG.md | 34 ++++ CLAUDE.md | 11 +- README.md | 33 ++-- crates/clawhdf5-filters/Cargo.toml | 8 +- crates/clawhdf5-filters/README.md | 10 +- crates/clawhdf5-filters/src/fast_deflate.rs | 165 +++++++++++------ crates/clawhdf5-filters/src/lib.rs | 10 +- crates/clawhdf5-format/Cargo.toml | 10 +- crates/clawhdf5-format/src/filters.rs | 186 +++++++++++++++++--- crates/clawhdf5/Cargo.toml | 3 +- scripts/ci-test.sh | 33 ++++ 13 files changed, 471 insertions(+), 113 deletions(-) diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index dca3504..7b0b5c3 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -28,8 +28,9 @@ jobs: # dependency a failure (CLAWHDF5_REQUIRE_INTEROP below). run: | apt-get update - # cmake builds libz-ng-sys (clawhdf5-format's default `fast-deflate`); - # rust:latest does not ship it. + # cmake builds libz-ng-sys for the opt-in `fast-deflate` (zlib-ng) + # steps in ci-test.sh; rust:latest does not ship it. The default + # build (pure-Rust zlib-rs) does not need it. apt-get install -y --no-install-recommends python3 python3-venv cmake python3 -m venv /opt/interop /opt/interop/bin/pip install --no-cache-dir h5py numpy netCDF4 xarray @@ -75,19 +76,6 @@ jobs: command -v rustup >/dev/null || curl -sSf --retry 5 https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain none rustup toolchain install stable --profile minimal --component clippy echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" - - name: Build dependencies - # libz-ng-sys (clawhdf5-format's default `fast-deflate`) needs cmake. - # In the Docker runner the job is root and can install it; a host - # runner cannot, so say what is missing rather than fail inside a - # build script. - run: | - if command -v cmake >/dev/null; then cmake --version | head -1; exit 0; fi - if [ "$(id -u)" = 0 ]; then - apt-get update -qq && apt-get install -y -qq cmake - else - echo "::error::cmake is not installed on this runner host (needed by libz-ng-sys)" - exit 1 - fi - name: Confirm aarch64 run: | test "$(uname -m)" = aarch64 diff --git a/BENCHMARKS.md b/BENCHMARKS.md index 1d782e4..969d1ed 100644 --- a/BENCHMARKS.md +++ b/BENCHMARKS.md @@ -1185,6 +1185,69 @@ cargo run --release --bin ephemeral_perf --- +## Deflate backend: zlib-rs vs zlib-ng + +Measured 2026-09-23 on tank (AMD Ryzen 7 7800X3D, 8C/16T). The default +deflate backend is now **zlib-rs**, a pure-Rust port of zlib-ng; zlib-ng (C, +built with cmake) was the default before and is still available as +`fast-deflate`. Both builds were compiled once into separate target +directories and run **alternately, three rounds each**; figures are medians. + +```bash +# zlib-rs (default) +cargo bench -p clawhdf5-filters --bench deflate_bench +cargo bench -p clawhdf5-bench --bench h5bench_write --features libhdf5-compare -- '^write_2d_chunked/' +cargo run --release -p clawhdf5-bench --bin read_harness +# zlib-ng: add --features fast-deflate (filters) or clawhdf5-format/fast-deflate (bench) +``` + +| Workload | zlib-rs | zlib-ng | rs / ng | +|---|---:|---:|---:| +| HDF5 chunked write, deflate-6, 512×512 f32 | 1.458 ms | 1.484 ms | 0.98 | +| HDF5 chunked write, deflate-6, 128×128 f32 | 157.6 µs | 152.8 µs | 1.03 | +| HDF5 chunked write, deflate-6, 32×32 f32 | 62.9 µs | 60.2 µs | 1.05 | +| HDF5 read, 64 MB chunked + deflate, full | 64.4 ms | 65.2 ms | 0.99 | +| HDF5 read, 64×64 window (1 chunk) | 0.18 ms | 0.17 ms | 1.06 | +| HDF5 read, 512×512 window (4–9 chunks) | 4.10 ms | 4.10 ms | 1.00 | +| HDF5 read, one row / one column | 1.00 / 2.01 ms | 0.95 / 1.95 ms | 1.05 / 1.03 | +| Raw inflate, 8 MB f64 | 5.92 ms | 6.06 ms | 0.98 | +| Raw inflate, 1 MB sine | 82.8 µs | 68.6 µs | 1.21 | +| Raw deflate-6, 8 MB f64 / 1 MB sine | 92.2 / 2.01 ms | 83.1 / 1.84 ms | 1.11 / 1.09 | + +Compressed output is **byte-identical** between the two at levels 1, 6 and 9 +on all three inputs, so files do not change size. libhdf5 1.14.6 took 51.4 ms +for the 512×512 write in the same session (35× the zlib-rs figure). + +On the HDF5 paths zlib-rs is within 6% of zlib-ng everywhere, and ahead on the +largest write. The raw codec loops show zlib-ng still slightly faster at +compression (~10%), which chunked writes do not expose because encoding runs +in parallel across chunks. + +**Two findings along the way.** The first measurement had zlib-rs 1.2–1.9× +slower on single-chunk reads and 3.7× slower on a 1 MB inflate — slower even +than miniz_oxide. Neither was zlib-rs's fault: + +1. **Runtime CPU detection was off.** zlib-rs needs its `std` feature to + detect and use SIMD at runtime; flate2 turns it on through its default + `runtime_detection` feature, which our `default-features = false` flate2 + dependency was disabling. With it, a 1 MB inflate goes 282 → 83 µs. + `clawhdf5-format/zlib-rs` and `clawhdf5-filters/zlib-rs` now enable it. +2. **The codec was fed through a 32 KiB buffer.** Both deflate paths used + flate2's streaming `read::ZlibDecoder` / `write::ZlibEncoder`. A chunk's + decompressed size is known, so they now hand the codec the whole input in + one call, into an output buffer sized up front. Worth ~5% on chunked + writes and ~10% on zlib-ng's 1 MB inflate. It also closed a hole: the + streaming reader returned a truncated stream's bytes without an error, so + a truncated chunk read back short; it is now an error. + +| 1 MB inflate, same build otherwise | zlib-rs | zlib-ng | +|---|---:|---:| +| streaming reader, no runtime detection | 284.1 µs | 76.7 µs | +| one-shot, no runtime detection | 282.3 µs | 68.5 µs | +| one-shot + runtime detection (shipped) | **82.8 µs** | **68.6 µs** | + +--- + ## h5bench-Equivalent I/O Benchmarks Criterion harness mirroring h5bench serial workloads. clawhdf5 benchmarks dated 2026-07-01; diff --git a/CHANGELOG.md b/CHANGELOG.md index cdccb42..e437f17 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,14 @@ ## Unreleased ### Upgrade Notes +- **The default build no longer compiles any C.** Deflate now defaults to the + pure-Rust zlib-rs instead of zlib-ng, so building the core crates needs + neither cmake nor a C compiler. Speed on HDF5 reads and writes is within 6% + of zlib-ng, and compressed output is byte-identical. To keep zlib-ng, enable + `fast-deflate` (on `clawhdf5`, `clawhdf5-format` or `clawhdf5-filters`); it + overrides zlib-rs wherever it is on. +- **A truncated deflate chunk is now an error.** It used to read back short, + with no error. - **New stores use the int8 vector index by default.** `MemoryConfig::quantized_index` now defaults to `true`: a quarter of the index memory, builds 1.8x (x86-64) and 2.3x (Raspberry Pi 5) faster, and @@ -13,6 +21,28 @@ `quantized_index = false`, or pass `create --f32-index` to the CLI, to opt out. The CLI's `--quantized-index` is still accepted but is now a no-op. +### Build +- **Pure-Rust default.** `clawhdf5-format`, `clawhdf5-filters` and the + `clawhdf5` facade default to the `zlib-rs` deflate backend; `fast-deflate` + (zlib-ng) is opt-in. No crate in the default dependency tree of the core + crates compiles C, and `ci-test.sh` now fails if one appears. The facade's + `fast-deflate` was on by default and is now off. See `BENCHMARKS.md`, + "Deflate backend". +- `zlib-rs` also enables flate2's `runtime_detection`. Without it zlib-rs has + no `std`, cannot detect SIMD at runtime, and inflates 3.5x slower; the + workspace builds flate2 with `default-features = false`, which had been + switching it off. +- CI keeps zlib-ng building and tested; the arm64 job no longer needs cmake. + +### Correctness +- `clawhdf5-format`: **a truncated deflate chunk read back short, with no + error.** The deflate filter used flate2's streaming reader, which returns the + bytes it has when the input runs out before the end-of-stream marker. It now + decodes in one pass into a buffer sized to the chunk and reports a + truncated stream as `DecompressionError`. Same fix in `clawhdf5-filters`, + where output longer than the stated size was also silently cut off; it is + now an error. + ### Defaults - `clawhdf5-agent`: `MemoryConfig::quantized_index` defaults to `true` for new stores. The reason it had been off — that int8 search was slower on ARM — @@ -26,6 +56,10 @@ knew to ask; it now only ever switches the default off. ### Performance +- `clawhdf5-format`, `clawhdf5-filters`: both deflate paths hand the codec the + whole chunk in one call, into a buffer allocated once, instead of streaming + it through a 32 KiB buffer: about 5% on chunked writes and 10% on zlib-ng's + 1 MB inflate. - `clawhdf5-accel`: **`dot_i8` has aarch64 kernels** — `SDOT` for CPUs with the ARMv8.2 dot-product extension (Cortex-A76 and later, Neoverse-N1, every Apple Silicon generation) and plain NEON (`vmull_s8` + `vpadalq_s16`) for diff --git a/CLAUDE.md b/CLAUDE.md index 6165035..6b9fa24 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -27,7 +27,11 @@ Cargo workspace with 16 crates under `crates/` (plus `libaec-sys`, an internal F | `clawhdf5-bench` | Benchmark suite | ## Key Features -- Zero-dependency HDF5 read/write (no libhdf5 C library required) +- Zero-C-dependency HDF5 read/write: no libhdf5, and deflate defaults to + pure-Rust zlib-rs (`fast-deflate` opts into zlib-ng, which needs cmake). + `ci-test.sh` fails if a C-building crate enters the core crates' default + tree. flate2 must keep `runtime_detection` with zlib-rs — without it zlib-rs + loses SIMD and inflates 3.5x slower. - HNSW vector index for semantic similarity search over agent memories — the `clawhdf5-agent` `hnsw` feature is **on by default**, so `hybrid_search` uses the approximate `clawhdf5-ann` index for the vector stage (the index mirrors @@ -128,8 +132,9 @@ cargo test --workspace Keep workflows free of JavaScript actions (`actions/checkout`, `actions/cache`, …): `rust:latest` has no `node`, and not every runner reaches GitHub, where -they are fetched from. Check out with plain `git` instead. Both jobs need -`cmake` for `libz-ng-sys` (from `clawhdf5-format`'s default `fast-deflate`). +they are fetched from. Check out with plain `git` instead. The `test` job +installs `cmake` for the opt-in `fast-deflate` (zlib-ng) steps; the default +build needs no C toolchain, so `test-arm64` does not. All runners are on `gitea-runner` 3.5.0, from `docker.gitea.com/act_runner` — `gitea/act_runner:latest` on Docker Hub is frozen at 0.6.1. diff --git a/README.md b/README.md index 990b1c3..cf93565 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # ClawhDF5 -**The memory layer AI agents deserve. One file. Pure Rust. No libhdf5.** +**The memory layer AI agents deserve. One file. Pure Rust. Zero C dependencies.** [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE) [![Rust](https://img.shields.io/badge/rust-1.94%2B-orange.svg)](https://www.rust-lang.org) @@ -11,7 +11,7 @@ ClawHDF5 is a pure-Rust HDF5 implementation combined with a research-grade agent memory engine. It gives AI agents persistent, searchable, integrity-checked memory — all stored in a single portable file. > **Two things live here:** -> - **A general-purpose, pure-Rust HDF5 library** — no libhdf5, NetCDF-4 support, SIMD/GPU acceleration. See the **[Crate Map](#crate-map)** and **[BENCHMARKS.md](BENCHMARKS.md)** for the libhdf5 head-to-head numbers. +> - **A general-purpose, pure-Rust HDF5 library** — zero C dependencies, NetCDF-4 support, SIMD/GPU acceleration. See the **[Crate Map](#crate-map)** and **[BENCHMARKS.md](BENCHMARKS.md)** for the libhdf5 head-to-head numbers. > - **An agent memory layer built on top of it** — vector search, knowledge graph, hippocampal-style consolidation, in `clawhdf5-agent`. The crates are not on crates.io yet, so depend on them from git: @@ -22,10 +22,16 @@ clawhdf5 = { git = "https://git.redclaw.dev/quantumclaw/clawhdf5" } # cor clawhdf5-agent = { git = "https://git.redclaw.dev/quantumclaw/clawhdf5" } # + agent memory layer ``` -> **C dependencies, precisely:** the HDF5 format code is pure Rust and never -> links libhdf5. The default deflate backend is zlib-ng (`fast-deflate`), a C -> library built from source, so a default build needs `cmake` and a C -> compiler. Opt-in codecs (`zstd`, `szip`) and BLAS backends link C too. +> **C dependencies, precisely:** the core crates (`clawhdf5`, `clawhdf5-agent`, +> `-format`, `-io`, `-filters`, `-ann`, `-accel`, `-netcdf4`, `-cli`) build no C +> code by default — no libhdf5, and deflate is the pure-Rust +> [zlib-rs](https://github.com/trifectatechfoundation/zlib-rs), which matches +> zlib-ng on HDF5 reads and writes and produces byte-identical output +> ([BENCHMARKS.md § Deflate backend](BENCHMARKS.md#deflate-backend-zlib-rs-vs-zlib-ng)). +> CI fails if a C-building crate enters their default dependency tree. C comes +> in only when you ask for it: `fast-deflate` (zlib-ng, needs cmake), `zstd`, +> `szip`, the BLAS backends, `clawhdf5-migrate` (bundled SQLite) and the +> Node.js bindings. > **New here?** Start with the **[Quickstart Guide](docs/QUICKSTART.md)** · See **[Use Cases](docs/USE_CASES.md)** · Read **[Benchmarks](BENCHMARKS.md)** @@ -117,6 +123,12 @@ Figures below are from an independent reproduction run on a second machine (AMD | Sequential read (100K f32) | 23.3 µs | 63.6 µs | **2.7×** | | Sequential write (100K f32) | 210 µs | 189 µs | **≈ tie** | +The chunked-write row was re-measured on the same machine on 2026-09-23, after +the default deflate backend became pure-Rust zlib-rs: 1.46 ms against +libhdf5's 51.4 ms (**35×**), and 1.48 ms with zlib-ng. libhdf5's own time on +that machine moved from 65.0 to 51.4 ms between the two dates, which is most +of the difference from 45×; compare same-day numbers only. + ### Vector Search **HNSW (the default backend for `hybrid_search`)** — `search_harness`, clustered @@ -564,14 +576,15 @@ setting existed keep their `f32` index; opt out for new stores with | `deflate` | yes | Deflate compression | | `checksum` | yes | Jenkins lookup3 verification | | `provenance` | yes | SHA-256 provenance attributes | -| `fast-deflate` | **yes** | zlib-ng backend for faster deflate (C; needs `cmake`) | +| `zlib-rs` | **yes** | Pure-Rust deflate backend ([zlib-rs](https://github.com/trifectatechfoundation/zlib-rs)) | +| `fast-deflate` | no | zlib-ng deflate backend instead (C; needs `cmake`). Overrides `zlib-rs` when both are on | | `system-zlib-decompress` | **yes** | Use Apple's system libz for decompression (macOS only; no effect elsewhere) | | `parallel` | no | Parallel chunk encoding + compression (rayon) | | `fast-checksum` | no | crc32fast-accelerated checksums | | `lz4` | no | LZ4 block compression filter (id 32004) | | `zstd` | no | Zstandard compression filter (id 32015) | | `pcodec` | no | Pcodec lossless numerical codec (id 32023, via `pco` crate) | -| `system-zlib` / `zlib-rs` | no | Alternative zlib backends for deflate | +| `system-zlib` | no | System zlib backend for deflate (C) | | `blake3_hash` | no | BLAKE3 content hashing for provenance | | `szip` | no | SZIP filter (id 4) via libaec (C, through the internal `libaec-sys` crate) | @@ -601,7 +614,7 @@ setting existed keep their `f32` index; opt out for new stores with ## Building ```bash -# Default (needs cmake + a C compiler for zlib-ng) +# Default (pure Rust: no cmake or C compiler needed) cargo build --workspace # Agent memory with all accelerations (Linux) @@ -723,5 +736,5 @@ MIT

Built by RedClaw Systems
- ~86,000 lines of Rust. No libhdf5. One file to remember everything. + ~86,000 lines of Rust. Zero C dependencies. One file to remember everything.

diff --git a/crates/clawhdf5-filters/Cargo.toml b/crates/clawhdf5-filters/Cargo.toml index a2202b8..07fdeff 100644 --- a/crates/clawhdf5-filters/Cargo.toml +++ b/crates/clawhdf5-filters/Cargo.toml @@ -25,8 +25,12 @@ name = "compression_bench" harness = false [features] -default = ["fast-deflate"] +# Pure-Rust zlib-rs by default; `fast-deflate` (zlib-ng, C) overrides it. +default = ["zlib-rs"] fast-deflate = ["flate2/zlib-ng"] system-zlib = ["flate2/zlib-default"] -zlib-rs = ["flate2/zlib-rs"] +# `runtime_detection` gives zlib-rs `std`, which it needs to detect and use +# SIMD at runtime. flate2 enables it by default, but we build flate2 with +# default-features = false, and without it zlib-rs inflates 3.5x slower. +zlib-rs = ["flate2/zlib-rs", "flate2/runtime_detection"] apple-compression = [] diff --git a/crates/clawhdf5-filters/README.md b/crates/clawhdf5-filters/README.md index 4ccd3fd..45fcf30 100644 --- a/crates/clawhdf5-filters/README.md +++ b/crates/clawhdf5-filters/README.md @@ -8,16 +8,18 @@ Filter and compression pipeline for clawhdf5. ## Features - DEFLATE compression/decompression -- Fast deflate via zlib-ng (`fast-deflate` feature) +- Pure-Rust deflate via zlib-rs (default, `zlib-rs` feature) +- zlib-ng instead, if you want it (`fast-deflate` feature; C, needs cmake) - Apple Compression framework support (`apple-compression` feature) ## Usage ```rust -use clawhdf5_filters::{deflate_decode, deflate_encode}; +use clawhdf5_filters::{deflate_compress, deflate_decompress}; -let compressed = deflate_encode(&data, 6).unwrap(); -let decompressed = deflate_decode(&compressed).unwrap(); +let compressed = deflate_compress(&data, 6).unwrap(); +// The second argument bounds the output: the expected decompressed size. +let decompressed = deflate_decompress(&compressed, data.len()).unwrap(); ``` ## License diff --git a/crates/clawhdf5-filters/src/fast_deflate.rs b/crates/clawhdf5-filters/src/fast_deflate.rs index 6b4c65a..6ce268f 100644 --- a/crates/clawhdf5-filters/src/fast_deflate.rs +++ b/crates/clawhdf5-filters/src/fast_deflate.rs @@ -1,12 +1,13 @@ -//! Fast deflate backends: Apple Compression Framework and zlib-ng. +//! Deflate backends: Apple Compression Framework, zlib-ng and zlib-rs. //! //! Backend selection priority (decompression & compression): //! 1. Apple Compression Framework (macOS only, `apple-compression` feature) -//! 2. flate2 with zlib-ng backend (`fast-deflate` feature) or miniz_oxide (default) +//! 2. flate2 with zlib-ng (`fast-deflate`), else zlib-rs (`zlib-rs`, the +//! default), else miniz_oxide //! //! The Apple Compression Framework uses hardware-accelerated zlib on Apple Silicon -//! and is typically the fastest option on macOS. zlib-ng is the fastest portable -//! option and what C HDF5 uses internally. +//! and is typically the fastest option on macOS. zlib-rs is a pure-Rust port of +//! zlib-ng; see `BENCHMARKS.md` for how the two compare. // --------------------------------------------------------------------------- // Apple Compression Framework FFI (macOS only) @@ -243,65 +244,117 @@ mod apple { } // --------------------------------------------------------------------------- -// Streaming decompression via flate2 (uses zlib-ng when fast-deflate enabled) +// One-shot (de)compression via flate2 (whichever backend flate2 was built with) +// +// The whole input goes to the codec in one call, into an output buffer sized +// up front. `flate2::read::ZlibDecoder` / `write::ZlibEncoder` stream through a +// 32 KiB buffer instead, which cost zlib-rs up to 3.7x against zlib-ng on a +// 1 MB chunk. clawhdf5-format's deflate filter does the same; see +// `BENCHMARKS.md`, "Deflate backend". // --------------------------------------------------------------------------- -/// Streaming decompress with pre-allocated output buffer. -/// -/// When the output size is known (typical for HDF5 chunks), this avoids -/// dynamic reallocation by writing directly into a pre-sized buffer. +/// Decompress into a buffer pre-sized to `output_size`, the expected +/// decompressed length (known for HDF5 chunks). Output longer than that is an +/// error, as is a stream that ends early. pub(crate) fn flate2_decompress_preallocated( data: &[u8], output_size: usize, ) -> Result, String> { - use std::io::Read; - let mut decoder = flate2::read::ZlibDecoder::new(data); - let mut output = vec![0u8; output_size]; - let mut total_read = 0; - - loop { - match decoder.read(&mut output[total_read..]) { - Ok(0) => break, - Ok(n) => total_read += n, - Err(e) => return Err(e.to_string()), - } - } - output.truncate(total_read); - Ok(output) + inflate_bounded(data, output_size, output_size) } /// Absolute ceiling on decompressed output when the caller has no size hint, /// preventing unbounded allocation from a hostile/corrupted zlib stream. const MAX_DECOMPRESS_SIZE: usize = 256 * 1024 * 1024; -/// Streaming decompress with dynamic sizing (when output size is unknown). -/// -/// Bounded by [`MAX_DECOMPRESS_SIZE`] since there is no chunk-size hint to -/// validate against here — an unbounded `read_to_end` would let a hostile -/// zlib stream force arbitrarily large allocation (a "zlib bomb"). +/// Decompress with no size hint, bounded by [`MAX_DECOMPRESS_SIZE`] so a +/// hostile zlib stream cannot force arbitrarily large allocation (a "zlib +/// bomb"). pub(crate) fn flate2_decompress_streaming(data: &[u8]) -> Result, String> { - use std::io::Read; - let decoder = flate2::read::ZlibDecoder::new(data); - let mut result = Vec::new(); - decoder - .take(MAX_DECOMPRESS_SIZE as u64 + 1) - .read_to_end(&mut result) - .map_err(|e| e.to_string())?; - if result.len() > MAX_DECOMPRESS_SIZE { - return Err(format!( - "decompressed output exceeds {} MiB limit", - MAX_DECOMPRESS_SIZE / 1024 / 1024 - )); - } - Ok(result) + let hint = data.len().saturating_mul(4).min(1 << 20); + inflate_bounded(data, hint, MAX_DECOMPRESS_SIZE).map_err(|e| { + if e.ends_with("exceeds size limit") { + format!( + "decompressed output exceeds {} MiB limit", + MAX_DECOMPRESS_SIZE / 1024 / 1024 + ) + } else { + e + } + }) } -/// Compress data using flate2 (zlib-ng when fast-deflate enabled, else miniz_oxide). +/// Inflate a zlib stream, starting from `size_hint` bytes of output and +/// failing past `limit`. +fn inflate_bounded(data: &[u8], size_hint: usize, limit: usize) -> Result, String> { + use flate2::{Decompress, FlushDecompress, Status}; + + // One byte of headroom past the limit distinguishes an over-size stream + // from one that legitimately ends exactly at the limit. + let max_capacity = limit.saturating_add(1); + let mut out = Vec::new(); + out.try_reserve_exact(size_hint.clamp(1, max_capacity)) + .map_err(|e| format!("deflate: cannot allocate output: {e}"))?; + + let mut inflater = Decompress::new(true); + loop { + let (in_before, out_before) = (inflater.total_in(), inflater.total_out()); + let status = inflater + .decompress_vec( + &data[in_before as usize..], + &mut out, + FlushDecompress::Finish, + ) + .map_err(|e| format!("deflate: {e}"))?; + if out.len() > limit { + return Err("deflate: output exceeds size limit".into()); + } + match status { + Status::StreamEnd => return Ok(out), + Status::Ok | Status::BufError if out.len() == out.capacity() => { + let grow = out.capacity().min(max_capacity - out.capacity()).max(1); + out.try_reserve_exact(grow) + .map_err(|e| format!("deflate: cannot allocate output: {e}"))?; + } + Status::Ok | Status::BufError => { + if inflater.total_in() as usize >= data.len() + || (inflater.total_in(), inflater.total_out()) == (in_before, out_before) + { + return Err("deflate: truncated stream".into()); + } + } + } + } +} + +/// Compress data using flate2 (zlib-ng, zlib-rs or miniz_oxide; see module docs). pub(crate) fn flate2_compress(data: &[u8], level: u32) -> Result, String> { - use std::io::Write; - let mut encoder = flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::new(level)); - encoder.write_all(data).map_err(|e| e.to_string())?; - encoder.finish().map_err(|e| e.to_string()) + use flate2::{Compress, Compression, FlushCompress, Status}; + + // zlib's compressBound, plus the zlib header and trailer. + let bound = data.len() + (data.len() >> 12) + (data.len() >> 14) + (data.len() >> 25) + 13 + 6; + let mut out = Vec::new(); + out.try_reserve_exact(bound) + .map_err(|e| format!("deflate: cannot allocate output: {e}"))?; + + let mut deflater = Compress::new(Compression::new(level), true); + loop { + let (in_before, out_before) = (deflater.total_in(), deflater.total_out()); + let status = deflater + .compress_vec(&data[in_before as usize..], &mut out, FlushCompress::Finish) + .map_err(|e| format!("deflate: {e}"))?; + match status { + Status::StreamEnd => return Ok(out), + Status::Ok | Status::BufError if out.len() == out.capacity() => out + .try_reserve(out.capacity().max(4096)) + .map_err(|e| format!("deflate: cannot allocate output: {e}"))?, + Status::Ok | Status::BufError => { + if (deflater.total_in(), deflater.total_out()) == (in_before, out_before) { + return Err("deflate: encoder made no progress".into()); + } + } + } + } } // --------------------------------------------------------------------------- @@ -312,7 +365,7 @@ pub(crate) fn flate2_compress(data: &[u8], level: u32) -> Result, String /// /// Selection order: /// 1. Apple Compression Framework (macOS + `apple-compression` feature) -/// 2. flate2 (zlib-ng with `fast-deflate`, otherwise miniz_oxide) +/// 2. flate2 (zlib-ng with `fast-deflate`, else zlib-rs, else miniz_oxide) /// /// When `output_hint` > 0, pre-allocates the output buffer for zero-copy /// decompression (avoids reallocation). @@ -344,7 +397,7 @@ pub fn decompress(data: &[u8], output_hint: usize) -> Result, String> { /// /// Selection order: /// 1. Apple Compression Framework (macOS + `apple-compression` feature) -/// 2. flate2 (zlib-ng with `fast-deflate`, otherwise miniz_oxide) +/// 2. flate2 (zlib-ng with `fast-deflate`, else zlib-rs, else miniz_oxide) pub fn compress(data: &[u8], level: u32) -> Result, String> { #[cfg(all(target_os = "macos", feature = "apple-compression"))] { @@ -377,9 +430,19 @@ pub fn active_backend() -> &'static str { { "zlib-ng" } + // flate2 prefers a C zlib over zlib-rs when both are enabled. + #[cfg(all( + not(all(target_os = "macos", feature = "apple-compression")), + not(feature = "fast-deflate"), + feature = "zlib-rs" + ))] + { + "zlib-rs" + } #[cfg(not(any( all(target_os = "macos", feature = "apple-compression"), - feature = "fast-deflate" + feature = "fast-deflate", + feature = "zlib-rs" )))] { "miniz_oxide" @@ -436,7 +499,7 @@ mod tests { fn backend_name_is_set() { let name = active_backend(); assert!( - ["miniz_oxide", "zlib-ng", "apple-compression"].contains(&name), + ["miniz_oxide", "zlib-rs", "zlib-ng", "apple-compression"].contains(&name), "unexpected backend: {name}" ); } diff --git a/crates/clawhdf5-filters/src/lib.rs b/crates/clawhdf5-filters/src/lib.rs index ae67589..d9d18eb 100644 --- a/crates/clawhdf5-filters/src/lib.rs +++ b/crates/clawhdf5-filters/src/lib.rs @@ -2,12 +2,14 @@ //! //! Provides deflate (zlib) decompression/compression with multiple backend options: //! -//! - **Default**: `miniz_oxide` (pure Rust, no C dependencies) -//! - **`fast-deflate` feature**: `zlib-ng` via flate2 (~2-3x faster, matches C HDF5) +//! - **Default (`zlib-rs` feature)**: `zlib-rs` via flate2 (pure Rust, no C +//! dependencies) +//! - **`fast-deflate` feature**: `zlib-ng` via flate2 (C, built with cmake) //! - **`apple-compression` feature**: Apple Compression Framework on macOS //! (hardware-accelerated on Apple Silicon) +//! - With none of the above: `miniz_oxide` (pure Rust, slower) //! -//! Backend priority: apple-compression > zlib-ng > miniz_oxide. +//! Backend priority: apple-compression > zlib-ng > zlib-rs > miniz_oxide. pub mod fast_deflate; @@ -115,7 +117,7 @@ mod tests { fn backend_reports_name() { let name = deflate_backend(); assert!( - ["miniz_oxide", "zlib-ng", "apple-compression"].contains(&name), + ["miniz_oxide", "zlib-rs", "zlib-ng", "apple-compression"].contains(&name), "unexpected backend: {name}" ); } diff --git a/crates/clawhdf5-format/Cargo.toml b/crates/clawhdf5-format/Cargo.toml index 7263497..fe38392 100644 --- a/crates/clawhdf5-format/Cargo.toml +++ b/crates/clawhdf5-format/Cargo.toml @@ -32,7 +32,10 @@ name = "bench" harness = false [features] -default = ["std", "checksum", "deflate", "provenance", "fast-deflate", "system-zlib-decompress"] +# Deflate backend: `zlib-rs` (pure Rust) by default. `fast-deflate` selects +# zlib-ng instead (C, built with cmake); flate2 prefers a C zlib whenever one +# is enabled, so turning it on anywhere in the build overrides the default. +default = ["std", "checksum", "deflate", "provenance", "zlib-rs", "system-zlib-decompress"] std = [] checksum = [] deflate = ["flate2"] @@ -42,7 +45,10 @@ fast-checksum = ["crc32fast"] fast-deflate = ["flate2/zlib-ng"] system-zlib = ["flate2/zlib-default"] system-zlib-decompress = [] -zlib-rs = ["flate2/zlib-rs"] +# `runtime_detection` gives zlib-rs `std`, which it needs to detect and use +# SIMD at runtime. flate2 enables it by default, but we build flate2 with +# default-features = false, and without it zlib-rs inflates 3.5x slower. +zlib-rs = ["flate2/zlib-rs", "flate2/runtime_detection"] lz4 = ["lz4_flex"] zstd = ["dep:zstd"] blake3_hash = ["blake3"] diff --git a/crates/clawhdf5-format/src/filters.rs b/crates/clawhdf5-format/src/filters.rs index 9979d3f..c10cea9 100644 --- a/crates/clawhdf5-format/src/filters.rs +++ b/crates/clawhdf5-format/src/filters.rs @@ -629,21 +629,70 @@ fn deflate_decompress(data: &[u8], expected_bytes: usize) -> Result, For // Fall through to flate2 on error } - use std::io::Read; - let decoder = flate2::read::ZlibDecoder::new(data); - let mut result = Vec::with_capacity(limit.min(1 << 20)); - // Read one byte past the limit so an over-size stream is distinguishable + // A chunk's decompressed size is known, so allocate it once; without one, + // start from a multiple of the input and grow. + let size_hint = if expected_bytes != 0 { + expected_bytes + } else { + data.len().saturating_mul(4).min(1 << 20) + }; + inflate_bounded(data, size_hint, limit).map_err(FormatError::DecompressionError) +} + +/// Inflate a zlib stream into a buffer sized up front, handing the decoder the +/// whole input at once. +/// +/// `flate2::read::ZlibDecoder` feeds its input through a 32 KiB buffer and +/// grows the output as it goes; on single chunks that cost zlib-rs up to 3.7x +/// against zlib-ng (`BENCHMARKS.md`, "Deflate backend"). Output beyond `limit` +/// is an error, as is a stream that ends before its end-of-stream marker (the +/// streaming reader returned the bytes it had and no error). +#[cfg(feature = "deflate")] +pub(crate) fn inflate_bounded( + data: &[u8], + size_hint: usize, + limit: usize, +) -> Result, String> { + use flate2::{Decompress, FlushDecompress, Status}; + + // One byte of headroom past the limit distinguishes an over-size stream // from one that legitimately ends exactly at the limit. - decoder - .take(limit as u64 + 1) - .read_to_end(&mut result) - .map_err(|e| FormatError::DecompressionError(e.to_string()))?; - if result.len() > limit { - return Err(FormatError::DecompressionError( - "deflate: output exceeds size limit".into(), - )); + let max_capacity = limit.saturating_add(1); + let mut out = Vec::new(); + out.try_reserve_exact(size_hint.clamp(1, max_capacity)) + .map_err(|e| format!("deflate: cannot allocate output: {e}"))?; + + let mut inflater = Decompress::new(true); + loop { + let (in_before, out_before) = (inflater.total_in(), inflater.total_out()); + let status = inflater + .decompress_vec( + &data[in_before as usize..], + &mut out, + FlushDecompress::Finish, + ) + .map_err(|e| format!("deflate: {e}"))?; + if out.len() > limit { + return Err("deflate: output exceeds size limit".into()); + } + match status { + Status::StreamEnd => return Ok(out), + Status::Ok | Status::BufError if out.len() == out.capacity() => { + // Out of room: double, up to the limit. + let grow = out.capacity().min(max_capacity - out.capacity()).max(1); + out.try_reserve_exact(grow) + .map_err(|e| format!("deflate: cannot allocate output: {e}"))?; + } + Status::Ok | Status::BufError => { + // Room left, so the decoder stopped for want of input. + if inflater.total_in() as usize >= data.len() + || (inflater.total_in(), inflater.total_out()) == (in_before, out_before) + { + return Err("deflate: truncated stream".into()); + } + } + } } - Ok(result) } /// Direct FFI to Apple's system libz for fast decompression. @@ -722,14 +771,41 @@ fn deflate_decompress(_data: &[u8], _expected_bytes: usize) -> Result, F /// Compress data with zlib. #[cfg(feature = "deflate")] fn deflate_compress(data: &[u8], level: u32) -> Result, FormatError> { - use std::io::Write; - let mut encoder = flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::new(level)); - encoder - .write_all(data) - .map_err(|e| FormatError::CompressionError(e.to_string()))?; - encoder - .finish() - .map_err(|e| FormatError::CompressionError(e.to_string())) + deflate_bounded(data, level).map_err(FormatError::CompressionError) +} + +/// Deflate `data` into a zlib stream in one pass, into a buffer sized for the +/// worst case up front (the same reasoning as [`inflate_bounded`]). +#[cfg(feature = "deflate")] +pub(crate) fn deflate_bounded(data: &[u8], level: u32) -> Result, String> { + use flate2::{Compress, Compression, FlushCompress, Status}; + + // zlib's compressBound, plus the zlib header and trailer. + let bound = data.len() + (data.len() >> 12) + (data.len() >> 14) + (data.len() >> 25) + 13 + 6; + let mut out = Vec::new(); + out.try_reserve_exact(bound) + .map_err(|e| format!("deflate: cannot allocate output: {e}"))?; + + let mut deflater = Compress::new(Compression::new(level), true); + loop { + let (in_before, out_before) = (deflater.total_in(), deflater.total_out()); + let status = deflater + .compress_vec(&data[in_before as usize..], &mut out, FlushCompress::Finish) + .map_err(|e| format!("deflate: {e}"))?; + match status { + Status::StreamEnd => return Ok(out), + // The bound should make running out of room unreachable; grow + // rather than fail if it happens. + Status::Ok | Status::BufError if out.len() == out.capacity() => out + .try_reserve(out.capacity().max(4096)) + .map_err(|e| format!("deflate: cannot allocate output: {e}"))?, + Status::Ok | Status::BufError => { + if (deflater.total_in(), deflater.total_out()) == (in_before, out_before) { + return Err("deflate: encoder made no progress".into()); + } + } + } + } } #[cfg(not(feature = "deflate"))] @@ -1833,6 +1909,74 @@ mod tests { assert!(deflate_decompress(&compressed, 64).is_err()); } + #[cfg(feature = "deflate")] + fn noisy_bytes(n: usize) -> Vec { + // Compressible but not trivially so. + (0..n) + .map(|i| ((i as f64 * 0.01).sin() * 127.0 + 128.0) as u8 ^ (i as u8 & 3)) + .collect() + } + + #[test] + #[cfg(feature = "deflate")] + fn deflate_decompress_accepts_output_exactly_at_chunk_size() { + let data = noisy_bytes(100_000); + let compressed = deflate_compress(&data, 6).unwrap(); + assert_eq!(deflate_decompress(&compressed, data.len()).unwrap(), data); + // One byte short of the real size is over the limit. + assert!(deflate_decompress(&compressed, data.len() - 1).is_err()); + } + + #[test] + #[cfg(feature = "deflate")] + fn deflate_decompress_without_size_grows_the_buffer() { + // No chunk size: the output starts at 4x the input and has to grow. + let data = vec![7u8; 3 * 1024 * 1024]; + let compressed = deflate_compress(&data, 6).unwrap(); + assert!(compressed.len() * 4 < data.len()); + assert_eq!(deflate_decompress(&compressed, 0).unwrap(), data); + } + + #[test] + #[cfg(feature = "deflate")] + fn deflate_decompress_rejects_truncated_stream() { + // The streaming reader this replaced returned the bytes it had and no + // error, so a truncated chunk read back short. + let data = noisy_bytes(100_000); + let compressed = deflate_compress(&data, 6).unwrap(); + for cut in [compressed.len() - 1, compressed.len() / 2, 3] { + assert!( + deflate_decompress(&compressed[..cut], data.len()).is_err(), + "truncated to {cut} of {} bytes", + compressed.len() + ); + } + } + + #[test] + #[cfg(feature = "deflate")] + fn deflate_compress_roundtrips_incompressible_data() { + // Random-looking input compresses to slightly more than it started + // as; the output must still fit the pre-sized buffer (or grow). + let mut x = 0x9E37_79B9_7F4A_7C15u64; + let data: Vec = (0..200_000) + .map(|_| { + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + x as u8 + }) + .collect(); + for level in [0, 1, 6, 9] { + let compressed = deflate_compress(&data, level).unwrap(); + assert_eq!(deflate_decompress(&compressed, data.len()).unwrap(), data); + } + assert_eq!( + deflate_decompress(&deflate_compress(&[], 6).unwrap(), 0).unwrap(), + Vec::::new() + ); + } + #[test] #[cfg(feature = "zstd")] fn zstd_decompress_rejects_output_exceeding_chunk_size() { diff --git a/crates/clawhdf5/Cargo.toml b/crates/clawhdf5/Cargo.toml index ee5326f..40dbec2 100644 --- a/crates/clawhdf5/Cargo.toml +++ b/crates/clawhdf5/Cargo.toml @@ -30,9 +30,10 @@ name = "parallel_bench" harness = false [features] -default = ["mmap", "fast-deflate", "provenance"] +default = ["mmap", "provenance"] mmap = ["clawhdf5-io/mmap"] parallel = ["clawhdf5-format/parallel", "rayon"] +# zlib-ng (C, needs cmake) instead of the default pure-Rust zlib-rs. fast-deflate = ["clawhdf5-format/fast-deflate"] apple-compression = [] zstd = ["clawhdf5-format/zstd"] diff --git a/scripts/ci-test.sh b/scripts/ci-test.sh index f04dd09..4df9e33 100755 --- a/scripts/ci-test.sh +++ b/scripts/ci-test.sh @@ -77,6 +77,35 @@ run_step "cargo clippy (ann parallel)" cargo clippy \ --features parallel \ -- -D warnings +# zlib-ng is opt-in (`fast-deflate`; the default is pure-Rust zlib-rs), so +# nothing above builds it. Keep it compiling and passing. +run_step "cargo clippy (fast-deflate / zlib-ng)" cargo clippy \ + -p clawhdf5-format -p clawhdf5-filters -p clawhdf5 \ + --all-targets \ + --features clawhdf5-format/fast-deflate,clawhdf5-filters/fast-deflate \ + -- -D warnings + +# The README promises that the core crates build no C by default. Hold it to +# that: fail if a crate that compiles C (a *-sys crate, cc or cmake) enters the +# default dependency tree of any of them. clawhdf5-migrate (bundled SQLite), +# clawhdf5-napi (Node) and clawhdf5-gpu (graphics drivers) are exempt. +no_c_in_default_build() { + local crate found=0 + for crate in clawhdf5-format clawhdf5-io clawhdf5-filters clawhdf5 \ + clawhdf5-agent clawhdf5-ann clawhdf5-accel clawhdf5-netcdf4 clawhdf5-cli; do + local c_deps + c_deps=$(cargo tree -q -p "$crate" -e normal,build --prefix none \ + | grep -E '^([a-z0-9_-]+-sys|cc|cmake) v' | sort -u) + if [ -n "$c_deps" ]; then + echo "$crate pulls in C by default:" + echo "$c_deps" | sed 's/^/ /' + found=1 + fi + done + return $found +} +run_step "no C in the default build (core crates)" no_c_in_default_build + # 4. Tests (exclude clawhdf5-py) run_step "cargo test" cargo test \ --workspace \ @@ -90,6 +119,10 @@ run_step "cargo test (ann parallel)" cargo test \ -p clawhdf5-ann \ --features parallel +run_step "cargo test (fast-deflate / zlib-ng)" cargo test \ + -p clawhdf5-format -p clawhdf5-filters -p clawhdf5 \ + --features clawhdf5-format/fast-deflate,clawhdf5-filters/fast-deflate + # 5. Python interop suites. The h5py writer tests are #[ignore]d so a plain # `cargo test` stays hermetic; run them explicitly here. # On a PEP 668 "externally managed" system h5py can only live in a -- 2.54.0 From fc7ae6549ac500e96d97c020a908038f4b838923 Mon Sep 17 00:00:00 2001 From: osobh Date: Wed, 23 Sep 2026 11:05:26 -0500 Subject: [PATCH 3/3] build: declare Rust 1.92 as the MSRV and check it in CI rust-version = "1.92" in [workspace.package], inherited by every crate. 1.92 is the floor: wgpu (clawhdf5-gpu) requires it, and the whole workspace, Python bindings included, checks cleanly on it. ci-test.sh reads the version from Cargo.toml and checks the workspace on exactly that toolchain, so the manifests and the README badge cannot drift from what actually builds. The badge said 1.75, below edition 2024's own floor. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 4 ++++ CLAUDE.md | 3 ++- Cargo.toml | 3 +++ README.md | 2 +- crates/clawhdf5-accel/Cargo.toml | 1 + crates/clawhdf5-agent/Cargo.toml | 1 + crates/clawhdf5-android/Cargo.toml | 1 + crates/clawhdf5-ann/Cargo.toml | 1 + crates/clawhdf5-bench/Cargo.toml | 1 + crates/clawhdf5-cli/Cargo.toml | 1 + crates/clawhdf5-derive/Cargo.toml | 1 + crates/clawhdf5-filters/Cargo.toml | 1 + crates/clawhdf5-format/Cargo.toml | 1 + crates/clawhdf5-gpu/Cargo.toml | 1 + crates/clawhdf5-io/Cargo.toml | 1 + crates/clawhdf5-migrate/Cargo.toml | 1 + crates/clawhdf5-napi/Cargo.toml | 1 + crates/clawhdf5-netcdf4/Cargo.toml | 1 + crates/clawhdf5-py/Cargo.toml | 1 + crates/clawhdf5/Cargo.toml | 1 + crates/libaec-sys/Cargo.toml | 1 + scripts/ci-test.sh | 15 +++++++++++++++ 22 files changed, 42 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e437f17..5f8dc5b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ overrides zlib-rs wherever it is on. - **A truncated deflate chunk is now an error.** It used to read back short, with no error. +- **Minimum supported Rust is 1.92**, now declared in every crate's + `rust-version` and checked in CI. - **New stores use the int8 vector index by default.** `MemoryConfig::quantized_index` now defaults to `true`: a quarter of the index memory, builds 1.8x (x86-64) and 2.3x (Raspberry Pi 5) faster, and @@ -32,6 +34,8 @@ no `std`, cannot detect SIMD at runtime, and inflates 3.5x slower; the workspace builds flate2 with `default-features = false`, which had been switching it off. +- `rust-version = "1.92"` for the whole workspace (the floor: `wgpu` requires + it), and CI checks the workspace on exactly that toolchain. - CI keeps zlib-ng building and tested; the arm64 job no longer needs cmake. ### Correctness diff --git a/CLAUDE.md b/CLAUDE.md index 6b9fa24..cb10c9e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -31,7 +31,8 @@ Cargo workspace with 16 crates under `crates/` (plus `libaec-sys`, an internal F pure-Rust zlib-rs (`fast-deflate` opts into zlib-ng, which needs cmake). `ci-test.sh` fails if a C-building crate enters the core crates' default tree. flate2 must keep `runtime_detection` with zlib-rs — without it zlib-rs - loses SIMD and inflates 3.5x slower. + loses SIMD and inflates 3.5x slower. MSRV is 1.92 (`rust-version`, checked + in CI). - HNSW vector index for semantic similarity search over agent memories — the `clawhdf5-agent` `hnsw` feature is **on by default**, so `hybrid_search` uses the approximate `clawhdf5-ann` index for the vector stage (the index mirrors diff --git a/Cargo.toml b/Cargo.toml index 61c47f8..2e1f6fe 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,9 @@ resolver = "2" [workspace.package] version = "2.7.0" edition = "2024" +# Oldest toolchain that builds the whole workspace; CI checks it. wgpu (in +# clawhdf5-gpu) requires 1.92. +rust-version = "1.92" license = "MIT" repository = "https://git.redclaw.dev/quantumclaw/clawhdf5" diff --git a/README.md b/README.md index cf93565..58799af 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ **The memory layer AI agents deserve. One file. Pure Rust. Zero C dependencies.** [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE) -[![Rust](https://img.shields.io/badge/rust-1.94%2B-orange.svg)](https://www.rust-lang.org) +[![Rust](https://img.shields.io/badge/rust-1.92%2B-orange.svg)](https://www.rust-lang.org) [![Tests](https://img.shields.io/badge/tests-1850%2B-brightgreen.svg)](#building) [![LongMemEval](https://img.shields.io/badge/LongMemEval__s-Turn--Level%20Hit@5%2081.4%25%20hybrid-blue.svg)](BENCHMARKS.md#longmemeval-results) [![Footprint](https://img.shields.io/badge/on--disk-1.7%20KB%2Frecord-lightgrey.svg)](BENCHMARKS.md#memory-footprint-1) diff --git a/crates/clawhdf5-accel/Cargo.toml b/crates/clawhdf5-accel/Cargo.toml index b24d81d..e59a2ba 100644 --- a/crates/clawhdf5-accel/Cargo.toml +++ b/crates/clawhdf5-accel/Cargo.toml @@ -2,6 +2,7 @@ name = "clawhdf5-accel" version = "2.7.0" edition = "2024" +rust-version.workspace = true description = "SIMD-accelerated operations for rustyhdf5" license = "MIT" repository = "https://git.redclaw.dev/quantumclaw/clawhdf5" diff --git a/crates/clawhdf5-agent/Cargo.toml b/crates/clawhdf5-agent/Cargo.toml index a7e6796..b8d79ee 100644 --- a/crates/clawhdf5-agent/Cargo.toml +++ b/crates/clawhdf5-agent/Cargo.toml @@ -2,6 +2,7 @@ name = "clawhdf5-agent" version = "2.7.0" edition = "2024" +rust-version.workspace = true description = "HDF5-backed persistent memory store for on-device AI agents" license = "MIT" repository = "https://git.redclaw.dev/quantumclaw/clawhdf5" diff --git a/crates/clawhdf5-android/Cargo.toml b/crates/clawhdf5-android/Cargo.toml index 2ad609c..38dd2a0 100644 --- a/crates/clawhdf5-android/Cargo.toml +++ b/crates/clawhdf5-android/Cargo.toml @@ -2,6 +2,7 @@ name = "clawhdf5-android" version = "2.7.0" edition = "2024" +rust-version.workspace = true description = "Android JNI bridge for edgehdf5-memory HDF5 backend" license = "MIT" diff --git a/crates/clawhdf5-ann/Cargo.toml b/crates/clawhdf5-ann/Cargo.toml index 6a5469f..be1b37f 100644 --- a/crates/clawhdf5-ann/Cargo.toml +++ b/crates/clawhdf5-ann/Cargo.toml @@ -2,6 +2,7 @@ name = "clawhdf5-ann" version = "2.7.0" edition = "2024" +rust-version.workspace = true description = "HNSW approximate nearest neighbor index stored as HDF5" license = "MIT" repository = "https://git.redclaw.dev/quantumclaw/clawhdf5" diff --git a/crates/clawhdf5-bench/Cargo.toml b/crates/clawhdf5-bench/Cargo.toml index 879e921..5ac6350 100644 --- a/crates/clawhdf5-bench/Cargo.toml +++ b/crates/clawhdf5-bench/Cargo.toml @@ -2,6 +2,7 @@ name = "clawhdf5-bench" version = "2.7.0" edition = "2024" +rust-version.workspace = true description = "Benchmark harnesses for clawhdf5-agent (Track 8)" license = "MIT" diff --git a/crates/clawhdf5-cli/Cargo.toml b/crates/clawhdf5-cli/Cargo.toml index c622e6e..f8e7a15 100644 --- a/crates/clawhdf5-cli/Cargo.toml +++ b/crates/clawhdf5-cli/Cargo.toml @@ -2,6 +2,7 @@ name = "clawhdf5-cli" version = "2.7.0" edition = "2024" +rust-version.workspace = true license = "MIT" description = "CLI for clawhdf5 agent memory — create, save, search, recall, stats" repository = "https://git.redclaw.dev/quantumclaw/clawhdf5" diff --git a/crates/clawhdf5-derive/Cargo.toml b/crates/clawhdf5-derive/Cargo.toml index e15a46f..c5ca5f1 100644 --- a/crates/clawhdf5-derive/Cargo.toml +++ b/crates/clawhdf5-derive/Cargo.toml @@ -2,6 +2,7 @@ name = "clawhdf5-derive" version = "2.7.0" edition = "2024" +rust-version.workspace = true description = "Derive macros for rustyhdf5 HDF5 traits" license = "MIT" repository = "https://git.redclaw.dev/quantumclaw/clawhdf5" diff --git a/crates/clawhdf5-filters/Cargo.toml b/crates/clawhdf5-filters/Cargo.toml index 07fdeff..dd6060a 100644 --- a/crates/clawhdf5-filters/Cargo.toml +++ b/crates/clawhdf5-filters/Cargo.toml @@ -2,6 +2,7 @@ name = "clawhdf5-filters" version = "2.7.0" edition = "2024" +rust-version.workspace = true description = "Filter and compression pipeline for clawhdf5" license = "MIT" repository = "https://git.redclaw.dev/quantumclaw/clawhdf5" diff --git a/crates/clawhdf5-format/Cargo.toml b/crates/clawhdf5-format/Cargo.toml index fe38392..8567a7f 100644 --- a/crates/clawhdf5-format/Cargo.toml +++ b/crates/clawhdf5-format/Cargo.toml @@ -2,6 +2,7 @@ name = "clawhdf5-format" version = "2.7.0" edition = "2024" +rust-version.workspace = true description = "Pure-Rust HDF5 binary format parsing and writing — no C dependencies" license = "MIT" repository = "https://git.redclaw.dev/quantumclaw/clawhdf5" diff --git a/crates/clawhdf5-gpu/Cargo.toml b/crates/clawhdf5-gpu/Cargo.toml index 3ac02aa..d24aba4 100644 --- a/crates/clawhdf5-gpu/Cargo.toml +++ b/crates/clawhdf5-gpu/Cargo.toml @@ -2,6 +2,7 @@ name = "clawhdf5-gpu" version = "2.7.0" edition = "2024" +rust-version.workspace = true description = "GPU-accelerated vector operations for rustyhdf5 using wgpu compute shaders" license = "MIT" repository = "https://git.redclaw.dev/quantumclaw/clawhdf5" diff --git a/crates/clawhdf5-io/Cargo.toml b/crates/clawhdf5-io/Cargo.toml index f339c71..53d82f1 100644 --- a/crates/clawhdf5-io/Cargo.toml +++ b/crates/clawhdf5-io/Cargo.toml @@ -2,6 +2,7 @@ name = "clawhdf5-io" version = "2.7.0" edition = "2024" +rust-version.workspace = true description = "I/O abstraction layer for rustyhdf5" license = "MIT" repository = "https://git.redclaw.dev/quantumclaw/clawhdf5" diff --git a/crates/clawhdf5-migrate/Cargo.toml b/crates/clawhdf5-migrate/Cargo.toml index cf714fd..a518a22 100644 --- a/crates/clawhdf5-migrate/Cargo.toml +++ b/crates/clawhdf5-migrate/Cargo.toml @@ -2,6 +2,7 @@ name = "clawhdf5-migrate" version = "2.7.0" edition = "2024" +rust-version.workspace = true description = "CLI to migrate SQLite agent memory databases to HDF5 format" license = "MIT" repository = "https://git.redclaw.dev/quantumclaw/clawhdf5" diff --git a/crates/clawhdf5-napi/Cargo.toml b/crates/clawhdf5-napi/Cargo.toml index 28d8bda..c47403f 100644 --- a/crates/clawhdf5-napi/Cargo.toml +++ b/crates/clawhdf5-napi/Cargo.toml @@ -2,6 +2,7 @@ name = "clawhdf5-napi" version = "2.7.0" edition = "2024" +rust-version.workspace = true description = "Node.js native addon (napi-rs) exposing clawhdf5-agent to TypeScript/JavaScript" license = "MIT" repository = "https://git.redclaw.dev/quantumclaw/clawhdf5" diff --git a/crates/clawhdf5-netcdf4/Cargo.toml b/crates/clawhdf5-netcdf4/Cargo.toml index ad49148..6c38342 100644 --- a/crates/clawhdf5-netcdf4/Cargo.toml +++ b/crates/clawhdf5-netcdf4/Cargo.toml @@ -2,6 +2,7 @@ name = "clawhdf5-netcdf4" version = "2.7.0" edition = "2024" +rust-version.workspace = true description = "NetCDF-4 read support built on rustyhdf5 — pure Rust, no C dependencies" license = "MIT" repository = "https://git.redclaw.dev/quantumclaw/clawhdf5" diff --git a/crates/clawhdf5-py/Cargo.toml b/crates/clawhdf5-py/Cargo.toml index 8f3ea65..0d54a4f 100644 --- a/crates/clawhdf5-py/Cargo.toml +++ b/crates/clawhdf5-py/Cargo.toml @@ -2,6 +2,7 @@ name = "clawhdf5-py" version = "2.7.0" edition = "2024" +rust-version.workspace = true description = "Python bindings for rustyhdf5 — a pure-Rust HDF5 library" license = "MIT" repository = "https://git.redclaw.dev/quantumclaw/clawhdf5" diff --git a/crates/clawhdf5/Cargo.toml b/crates/clawhdf5/Cargo.toml index 40dbec2..2e2b4b4 100644 --- a/crates/clawhdf5/Cargo.toml +++ b/crates/clawhdf5/Cargo.toml @@ -2,6 +2,7 @@ name = "clawhdf5" version = "2.7.0" edition = "2024" +rust-version.workspace = true description = "Pure-Rust HDF5 reader/writer — no C dependencies" license = "MIT" repository = "https://git.redclaw.dev/quantumclaw/clawhdf5" diff --git a/crates/libaec-sys/Cargo.toml b/crates/libaec-sys/Cargo.toml index 788781a..75a7e8b 100644 --- a/crates/libaec-sys/Cargo.toml +++ b/crates/libaec-sys/Cargo.toml @@ -2,6 +2,7 @@ name = "libaec-sys" version = "0.1.0" edition = "2024" +rust-version.workspace = true links = "aec" [build-dependencies] diff --git a/scripts/ci-test.sh b/scripts/ci-test.sh index 4df9e33..b52c88b 100755 --- a/scripts/ci-test.sh +++ b/scripts/ci-test.sh @@ -106,6 +106,21 @@ no_c_in_default_build() { } run_step "no C in the default build (core crates)" no_c_in_default_build +# The workspace declares a minimum Rust version (rust-version in Cargo.toml); +# check that it really builds there, so the README badge and the manifests +# cannot drift from the truth. Separate target dir: a different toolchain +# would otherwise invalidate the main build. +msrv_check() { + local msrv + msrv=$(sed -n 's/^rust-version = "\(.*\)"/\1/p' "$SCRIPT_DIR/../Cargo.toml") + [ -n "$msrv" ] || { echo "no rust-version in Cargo.toml"; return 1; } + rustup toolchain install "$msrv" --profile minimal >/dev/null || return 1 + echo "checking with Rust $msrv" + CARGO_TARGET_DIR="$SCRIPT_DIR/../target/msrv" cargo "+$msrv" check \ + --workspace --exclude clawhdf5-py --all-targets +} +run_step "MSRV check" msrv_check + # 4. Tests (exclude clawhdf5-py) run_step "cargo test" cargo test \ --workspace \ -- 2.54.0