diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index dca3504..7b0b5c3 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -28,8 +28,9 @@ jobs: # dependency a failure (CLAWHDF5_REQUIRE_INTEROP below). run: | apt-get update - # cmake builds libz-ng-sys (clawhdf5-format's default `fast-deflate`); - # rust:latest does not ship it. + # cmake builds libz-ng-sys for the opt-in `fast-deflate` (zlib-ng) + # steps in ci-test.sh; rust:latest does not ship it. The default + # build (pure-Rust zlib-rs) does not need it. apt-get install -y --no-install-recommends python3 python3-venv cmake python3 -m venv /opt/interop /opt/interop/bin/pip install --no-cache-dir h5py numpy netCDF4 xarray @@ -75,19 +76,6 @@ jobs: command -v rustup >/dev/null || curl -sSf --retry 5 https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain none rustup toolchain install stable --profile minimal --component clippy echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" - - name: Build dependencies - # libz-ng-sys (clawhdf5-format's default `fast-deflate`) needs cmake. - # In the Docker runner the job is root and can install it; a host - # runner cannot, so say what is missing rather than fail inside a - # build script. - run: | - if command -v cmake >/dev/null; then cmake --version | head -1; exit 0; fi - if [ "$(id -u)" = 0 ]; then - apt-get update -qq && apt-get install -y -qq cmake - else - echo "::error::cmake is not installed on this runner host (needed by libz-ng-sys)" - exit 1 - fi - name: Confirm aarch64 run: | test "$(uname -m)" = aarch64 diff --git a/BENCHMARKS.md b/BENCHMARKS.md index 1d782e4..969d1ed 100644 --- a/BENCHMARKS.md +++ b/BENCHMARKS.md @@ -1185,6 +1185,69 @@ cargo run --release --bin ephemeral_perf --- +## Deflate backend: zlib-rs vs zlib-ng + +Measured 2026-09-23 on tank (AMD Ryzen 7 7800X3D, 8C/16T). The default +deflate backend is now **zlib-rs**, a pure-Rust port of zlib-ng; zlib-ng (C, +built with cmake) was the default before and is still available as +`fast-deflate`. Both builds were compiled once into separate target +directories and run **alternately, three rounds each**; figures are medians. + +```bash +# zlib-rs (default) +cargo bench -p clawhdf5-filters --bench deflate_bench +cargo bench -p clawhdf5-bench --bench h5bench_write --features libhdf5-compare -- '^write_2d_chunked/' +cargo run --release -p clawhdf5-bench --bin read_harness +# zlib-ng: add --features fast-deflate (filters) or clawhdf5-format/fast-deflate (bench) +``` + +| Workload | zlib-rs | zlib-ng | rs / ng | +|---|---:|---:|---:| +| HDF5 chunked write, deflate-6, 512×512 f32 | 1.458 ms | 1.484 ms | 0.98 | +| HDF5 chunked write, deflate-6, 128×128 f32 | 157.6 µs | 152.8 µs | 1.03 | +| HDF5 chunked write, deflate-6, 32×32 f32 | 62.9 µs | 60.2 µs | 1.05 | +| HDF5 read, 64 MB chunked + deflate, full | 64.4 ms | 65.2 ms | 0.99 | +| HDF5 read, 64×64 window (1 chunk) | 0.18 ms | 0.17 ms | 1.06 | +| HDF5 read, 512×512 window (4–9 chunks) | 4.10 ms | 4.10 ms | 1.00 | +| HDF5 read, one row / one column | 1.00 / 2.01 ms | 0.95 / 1.95 ms | 1.05 / 1.03 | +| Raw inflate, 8 MB f64 | 5.92 ms | 6.06 ms | 0.98 | +| Raw inflate, 1 MB sine | 82.8 µs | 68.6 µs | 1.21 | +| Raw deflate-6, 8 MB f64 / 1 MB sine | 92.2 / 2.01 ms | 83.1 / 1.84 ms | 1.11 / 1.09 | + +Compressed output is **byte-identical** between the two at levels 1, 6 and 9 +on all three inputs, so files do not change size. libhdf5 1.14.6 took 51.4 ms +for the 512×512 write in the same session (35× the zlib-rs figure). + +On the HDF5 paths zlib-rs is within 6% of zlib-ng everywhere, and ahead on the +largest write. The raw codec loops show zlib-ng still slightly faster at +compression (~10%), which chunked writes do not expose because encoding runs +in parallel across chunks. + +**Two findings along the way.** The first measurement had zlib-rs 1.2–1.9× +slower on single-chunk reads and 3.7× slower on a 1 MB inflate — slower even +than miniz_oxide. Neither was zlib-rs's fault: + +1. **Runtime CPU detection was off.** zlib-rs needs its `std` feature to + detect and use SIMD at runtime; flate2 turns it on through its default + `runtime_detection` feature, which our `default-features = false` flate2 + dependency was disabling. With it, a 1 MB inflate goes 282 → 83 µs. + `clawhdf5-format/zlib-rs` and `clawhdf5-filters/zlib-rs` now enable it. +2. **The codec was fed through a 32 KiB buffer.** Both deflate paths used + flate2's streaming `read::ZlibDecoder` / `write::ZlibEncoder`. A chunk's + decompressed size is known, so they now hand the codec the whole input in + one call, into an output buffer sized up front. Worth ~5% on chunked + writes and ~10% on zlib-ng's 1 MB inflate. It also closed a hole: the + streaming reader returned a truncated stream's bytes without an error, so + a truncated chunk read back short; it is now an error. + +| 1 MB inflate, same build otherwise | zlib-rs | zlib-ng | +|---|---:|---:| +| streaming reader, no runtime detection | 284.1 µs | 76.7 µs | +| one-shot, no runtime detection | 282.3 µs | 68.5 µs | +| one-shot + runtime detection (shipped) | **82.8 µs** | **68.6 µs** | + +--- + ## h5bench-Equivalent I/O Benchmarks Criterion harness mirroring h5bench serial workloads. clawhdf5 benchmarks dated 2026-07-01; diff --git a/CHANGELOG.md b/CHANGELOG.md index cdccb42..e437f17 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,14 @@ ## Unreleased ### Upgrade Notes +- **The default build no longer compiles any C.** Deflate now defaults to the + pure-Rust zlib-rs instead of zlib-ng, so building the core crates needs + neither cmake nor a C compiler. Speed on HDF5 reads and writes is within 6% + of zlib-ng, and compressed output is byte-identical. To keep zlib-ng, enable + `fast-deflate` (on `clawhdf5`, `clawhdf5-format` or `clawhdf5-filters`); it + overrides zlib-rs wherever it is on. +- **A truncated deflate chunk is now an error.** It used to read back short, + with no error. - **New stores use the int8 vector index by default.** `MemoryConfig::quantized_index` now defaults to `true`: a quarter of the index memory, builds 1.8x (x86-64) and 2.3x (Raspberry Pi 5) faster, and @@ -13,6 +21,28 @@ `quantized_index = false`, or pass `create --f32-index` to the CLI, to opt out. The CLI's `--quantized-index` is still accepted but is now a no-op. +### Build +- **Pure-Rust default.** `clawhdf5-format`, `clawhdf5-filters` and the + `clawhdf5` facade default to the `zlib-rs` deflate backend; `fast-deflate` + (zlib-ng) is opt-in. No crate in the default dependency tree of the core + crates compiles C, and `ci-test.sh` now fails if one appears. The facade's + `fast-deflate` was on by default and is now off. See `BENCHMARKS.md`, + "Deflate backend". +- `zlib-rs` also enables flate2's `runtime_detection`. Without it zlib-rs has + no `std`, cannot detect SIMD at runtime, and inflates 3.5x slower; the + workspace builds flate2 with `default-features = false`, which had been + switching it off. +- CI keeps zlib-ng building and tested; the arm64 job no longer needs cmake. + +### Correctness +- `clawhdf5-format`: **a truncated deflate chunk read back short, with no + error.** The deflate filter used flate2's streaming reader, which returns the + bytes it has when the input runs out before the end-of-stream marker. It now + decodes in one pass into a buffer sized to the chunk and reports a + truncated stream as `DecompressionError`. Same fix in `clawhdf5-filters`, + where output longer than the stated size was also silently cut off; it is + now an error. + ### Defaults - `clawhdf5-agent`: `MemoryConfig::quantized_index` defaults to `true` for new stores. The reason it had been off — that int8 search was slower on ARM — @@ -26,6 +56,10 @@ knew to ask; it now only ever switches the default off. ### Performance +- `clawhdf5-format`, `clawhdf5-filters`: both deflate paths hand the codec the + whole chunk in one call, into a buffer allocated once, instead of streaming + it through a 32 KiB buffer: about 5% on chunked writes and 10% on zlib-ng's + 1 MB inflate. - `clawhdf5-accel`: **`dot_i8` has aarch64 kernels** — `SDOT` for CPUs with the ARMv8.2 dot-product extension (Cortex-A76 and later, Neoverse-N1, every Apple Silicon generation) and plain NEON (`vmull_s8` + `vpadalq_s16`) for diff --git a/CLAUDE.md b/CLAUDE.md index 6165035..6b9fa24 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -27,7 +27,11 @@ Cargo workspace with 16 crates under `crates/` (plus `libaec-sys`, an internal F | `clawhdf5-bench` | Benchmark suite | ## Key Features -- Zero-dependency HDF5 read/write (no libhdf5 C library required) +- Zero-C-dependency HDF5 read/write: no libhdf5, and deflate defaults to + pure-Rust zlib-rs (`fast-deflate` opts into zlib-ng, which needs cmake). + `ci-test.sh` fails if a C-building crate enters the core crates' default + tree. flate2 must keep `runtime_detection` with zlib-rs — without it zlib-rs + loses SIMD and inflates 3.5x slower. - HNSW vector index for semantic similarity search over agent memories — the `clawhdf5-agent` `hnsw` feature is **on by default**, so `hybrid_search` uses the approximate `clawhdf5-ann` index for the vector stage (the index mirrors @@ -128,8 +132,9 @@ cargo test --workspace Keep workflows free of JavaScript actions (`actions/checkout`, `actions/cache`, …): `rust:latest` has no `node`, and not every runner reaches GitHub, where -they are fetched from. Check out with plain `git` instead. Both jobs need -`cmake` for `libz-ng-sys` (from `clawhdf5-format`'s default `fast-deflate`). +they are fetched from. Check out with plain `git` instead. The `test` job +installs `cmake` for the opt-in `fast-deflate` (zlib-ng) steps; the default +build needs no C toolchain, so `test-arm64` does not. All runners are on `gitea-runner` 3.5.0, from `docker.gitea.com/act_runner` — `gitea/act_runner:latest` on Docker Hub is frozen at 0.6.1. diff --git a/README.md b/README.md index 990b1c3..cf93565 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # ClawhDF5 -**The memory layer AI agents deserve. One file. Pure Rust. No libhdf5.** +**The memory layer AI agents deserve. One file. Pure Rust. Zero C dependencies.** [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE) [![Rust](https://img.shields.io/badge/rust-1.94%2B-orange.svg)](https://www.rust-lang.org) @@ -11,7 +11,7 @@ ClawHDF5 is a pure-Rust HDF5 implementation combined with a research-grade agent memory engine. It gives AI agents persistent, searchable, integrity-checked memory — all stored in a single portable file. > **Two things live here:** -> - **A general-purpose, pure-Rust HDF5 library** — no libhdf5, NetCDF-4 support, SIMD/GPU acceleration. See the **[Crate Map](#crate-map)** and **[BENCHMARKS.md](BENCHMARKS.md)** for the libhdf5 head-to-head numbers. +> - **A general-purpose, pure-Rust HDF5 library** — zero C dependencies, NetCDF-4 support, SIMD/GPU acceleration. See the **[Crate Map](#crate-map)** and **[BENCHMARKS.md](BENCHMARKS.md)** for the libhdf5 head-to-head numbers. > - **An agent memory layer built on top of it** — vector search, knowledge graph, hippocampal-style consolidation, in `clawhdf5-agent`. The crates are not on crates.io yet, so depend on them from git: @@ -22,10 +22,16 @@ clawhdf5 = { git = "https://git.redclaw.dev/quantumclaw/clawhdf5" } # cor clawhdf5-agent = { git = "https://git.redclaw.dev/quantumclaw/clawhdf5" } # + agent memory layer ``` -> **C dependencies, precisely:** the HDF5 format code is pure Rust and never -> links libhdf5. The default deflate backend is zlib-ng (`fast-deflate`), a C -> library built from source, so a default build needs `cmake` and a C -> compiler. Opt-in codecs (`zstd`, `szip`) and BLAS backends link C too. +> **C dependencies, precisely:** the core crates (`clawhdf5`, `clawhdf5-agent`, +> `-format`, `-io`, `-filters`, `-ann`, `-accel`, `-netcdf4`, `-cli`) build no C +> code by default — no libhdf5, and deflate is the pure-Rust +> [zlib-rs](https://github.com/trifectatechfoundation/zlib-rs), which matches +> zlib-ng on HDF5 reads and writes and produces byte-identical output +> ([BENCHMARKS.md § Deflate backend](BENCHMARKS.md#deflate-backend-zlib-rs-vs-zlib-ng)). +> CI fails if a C-building crate enters their default dependency tree. C comes +> in only when you ask for it: `fast-deflate` (zlib-ng, needs cmake), `zstd`, +> `szip`, the BLAS backends, `clawhdf5-migrate` (bundled SQLite) and the +> Node.js bindings. > **New here?** Start with the **[Quickstart Guide](docs/QUICKSTART.md)** · See **[Use Cases](docs/USE_CASES.md)** · Read **[Benchmarks](BENCHMARKS.md)** @@ -117,6 +123,12 @@ Figures below are from an independent reproduction run on a second machine (AMD | Sequential read (100K f32) | 23.3 µs | 63.6 µs | **2.7×** | | Sequential write (100K f32) | 210 µs | 189 µs | **≈ tie** | +The chunked-write row was re-measured on the same machine on 2026-09-23, after +the default deflate backend became pure-Rust zlib-rs: 1.46 ms against +libhdf5's 51.4 ms (**35×**), and 1.48 ms with zlib-ng. libhdf5's own time on +that machine moved from 65.0 to 51.4 ms between the two dates, which is most +of the difference from 45×; compare same-day numbers only. + ### Vector Search **HNSW (the default backend for `hybrid_search`)** — `search_harness`, clustered @@ -564,14 +576,15 @@ setting existed keep their `f32` index; opt out for new stores with | `deflate` | yes | Deflate compression | | `checksum` | yes | Jenkins lookup3 verification | | `provenance` | yes | SHA-256 provenance attributes | -| `fast-deflate` | **yes** | zlib-ng backend for faster deflate (C; needs `cmake`) | +| `zlib-rs` | **yes** | Pure-Rust deflate backend ([zlib-rs](https://github.com/trifectatechfoundation/zlib-rs)) | +| `fast-deflate` | no | zlib-ng deflate backend instead (C; needs `cmake`). Overrides `zlib-rs` when both are on | | `system-zlib-decompress` | **yes** | Use Apple's system libz for decompression (macOS only; no effect elsewhere) | | `parallel` | no | Parallel chunk encoding + compression (rayon) | | `fast-checksum` | no | crc32fast-accelerated checksums | | `lz4` | no | LZ4 block compression filter (id 32004) | | `zstd` | no | Zstandard compression filter (id 32015) | | `pcodec` | no | Pcodec lossless numerical codec (id 32023, via `pco` crate) | -| `system-zlib` / `zlib-rs` | no | Alternative zlib backends for deflate | +| `system-zlib` | no | System zlib backend for deflate (C) | | `blake3_hash` | no | BLAKE3 content hashing for provenance | | `szip` | no | SZIP filter (id 4) via libaec (C, through the internal `libaec-sys` crate) | @@ -601,7 +614,7 @@ setting existed keep their `f32` index; opt out for new stores with ## Building ```bash -# Default (needs cmake + a C compiler for zlib-ng) +# Default (pure Rust: no cmake or C compiler needed) cargo build --workspace # Agent memory with all accelerations (Linux) @@ -723,5 +736,5 @@ MIT

Built by RedClaw Systems
- ~86,000 lines of Rust. No libhdf5. One file to remember everything. + ~86,000 lines of Rust. Zero C dependencies. One file to remember everything.

diff --git a/crates/clawhdf5-filters/Cargo.toml b/crates/clawhdf5-filters/Cargo.toml index a2202b8..07fdeff 100644 --- a/crates/clawhdf5-filters/Cargo.toml +++ b/crates/clawhdf5-filters/Cargo.toml @@ -25,8 +25,12 @@ name = "compression_bench" harness = false [features] -default = ["fast-deflate"] +# Pure-Rust zlib-rs by default; `fast-deflate` (zlib-ng, C) overrides it. +default = ["zlib-rs"] fast-deflate = ["flate2/zlib-ng"] system-zlib = ["flate2/zlib-default"] -zlib-rs = ["flate2/zlib-rs"] +# `runtime_detection` gives zlib-rs `std`, which it needs to detect and use +# SIMD at runtime. flate2 enables it by default, but we build flate2 with +# default-features = false, and without it zlib-rs inflates 3.5x slower. +zlib-rs = ["flate2/zlib-rs", "flate2/runtime_detection"] apple-compression = [] diff --git a/crates/clawhdf5-filters/README.md b/crates/clawhdf5-filters/README.md index 4ccd3fd..45fcf30 100644 --- a/crates/clawhdf5-filters/README.md +++ b/crates/clawhdf5-filters/README.md @@ -8,16 +8,18 @@ Filter and compression pipeline for clawhdf5. ## Features - DEFLATE compression/decompression -- Fast deflate via zlib-ng (`fast-deflate` feature) +- Pure-Rust deflate via zlib-rs (default, `zlib-rs` feature) +- zlib-ng instead, if you want it (`fast-deflate` feature; C, needs cmake) - Apple Compression framework support (`apple-compression` feature) ## Usage ```rust -use clawhdf5_filters::{deflate_decode, deflate_encode}; +use clawhdf5_filters::{deflate_compress, deflate_decompress}; -let compressed = deflate_encode(&data, 6).unwrap(); -let decompressed = deflate_decode(&compressed).unwrap(); +let compressed = deflate_compress(&data, 6).unwrap(); +// The second argument bounds the output: the expected decompressed size. +let decompressed = deflate_decompress(&compressed, data.len()).unwrap(); ``` ## License diff --git a/crates/clawhdf5-filters/src/fast_deflate.rs b/crates/clawhdf5-filters/src/fast_deflate.rs index 6b4c65a..6ce268f 100644 --- a/crates/clawhdf5-filters/src/fast_deflate.rs +++ b/crates/clawhdf5-filters/src/fast_deflate.rs @@ -1,12 +1,13 @@ -//! Fast deflate backends: Apple Compression Framework and zlib-ng. +//! Deflate backends: Apple Compression Framework, zlib-ng and zlib-rs. //! //! Backend selection priority (decompression & compression): //! 1. Apple Compression Framework (macOS only, `apple-compression` feature) -//! 2. flate2 with zlib-ng backend (`fast-deflate` feature) or miniz_oxide (default) +//! 2. flate2 with zlib-ng (`fast-deflate`), else zlib-rs (`zlib-rs`, the +//! default), else miniz_oxide //! //! The Apple Compression Framework uses hardware-accelerated zlib on Apple Silicon -//! and is typically the fastest option on macOS. zlib-ng is the fastest portable -//! option and what C HDF5 uses internally. +//! and is typically the fastest option on macOS. zlib-rs is a pure-Rust port of +//! zlib-ng; see `BENCHMARKS.md` for how the two compare. // --------------------------------------------------------------------------- // Apple Compression Framework FFI (macOS only) @@ -243,65 +244,117 @@ mod apple { } // --------------------------------------------------------------------------- -// Streaming decompression via flate2 (uses zlib-ng when fast-deflate enabled) +// One-shot (de)compression via flate2 (whichever backend flate2 was built with) +// +// The whole input goes to the codec in one call, into an output buffer sized +// up front. `flate2::read::ZlibDecoder` / `write::ZlibEncoder` stream through a +// 32 KiB buffer instead, which cost zlib-rs up to 3.7x against zlib-ng on a +// 1 MB chunk. clawhdf5-format's deflate filter does the same; see +// `BENCHMARKS.md`, "Deflate backend". // --------------------------------------------------------------------------- -/// Streaming decompress with pre-allocated output buffer. -/// -/// When the output size is known (typical for HDF5 chunks), this avoids -/// dynamic reallocation by writing directly into a pre-sized buffer. +/// Decompress into a buffer pre-sized to `output_size`, the expected +/// decompressed length (known for HDF5 chunks). Output longer than that is an +/// error, as is a stream that ends early. pub(crate) fn flate2_decompress_preallocated( data: &[u8], output_size: usize, ) -> Result, String> { - use std::io::Read; - let mut decoder = flate2::read::ZlibDecoder::new(data); - let mut output = vec![0u8; output_size]; - let mut total_read = 0; - - loop { - match decoder.read(&mut output[total_read..]) { - Ok(0) => break, - Ok(n) => total_read += n, - Err(e) => return Err(e.to_string()), - } - } - output.truncate(total_read); - Ok(output) + inflate_bounded(data, output_size, output_size) } /// Absolute ceiling on decompressed output when the caller has no size hint, /// preventing unbounded allocation from a hostile/corrupted zlib stream. const MAX_DECOMPRESS_SIZE: usize = 256 * 1024 * 1024; -/// Streaming decompress with dynamic sizing (when output size is unknown). -/// -/// Bounded by [`MAX_DECOMPRESS_SIZE`] since there is no chunk-size hint to -/// validate against here — an unbounded `read_to_end` would let a hostile -/// zlib stream force arbitrarily large allocation (a "zlib bomb"). +/// Decompress with no size hint, bounded by [`MAX_DECOMPRESS_SIZE`] so a +/// hostile zlib stream cannot force arbitrarily large allocation (a "zlib +/// bomb"). pub(crate) fn flate2_decompress_streaming(data: &[u8]) -> Result, String> { - use std::io::Read; - let decoder = flate2::read::ZlibDecoder::new(data); - let mut result = Vec::new(); - decoder - .take(MAX_DECOMPRESS_SIZE as u64 + 1) - .read_to_end(&mut result) - .map_err(|e| e.to_string())?; - if result.len() > MAX_DECOMPRESS_SIZE { - return Err(format!( - "decompressed output exceeds {} MiB limit", - MAX_DECOMPRESS_SIZE / 1024 / 1024 - )); - } - Ok(result) + let hint = data.len().saturating_mul(4).min(1 << 20); + inflate_bounded(data, hint, MAX_DECOMPRESS_SIZE).map_err(|e| { + if e.ends_with("exceeds size limit") { + format!( + "decompressed output exceeds {} MiB limit", + MAX_DECOMPRESS_SIZE / 1024 / 1024 + ) + } else { + e + } + }) } -/// Compress data using flate2 (zlib-ng when fast-deflate enabled, else miniz_oxide). +/// Inflate a zlib stream, starting from `size_hint` bytes of output and +/// failing past `limit`. +fn inflate_bounded(data: &[u8], size_hint: usize, limit: usize) -> Result, String> { + use flate2::{Decompress, FlushDecompress, Status}; + + // One byte of headroom past the limit distinguishes an over-size stream + // from one that legitimately ends exactly at the limit. + let max_capacity = limit.saturating_add(1); + let mut out = Vec::new(); + out.try_reserve_exact(size_hint.clamp(1, max_capacity)) + .map_err(|e| format!("deflate: cannot allocate output: {e}"))?; + + let mut inflater = Decompress::new(true); + loop { + let (in_before, out_before) = (inflater.total_in(), inflater.total_out()); + let status = inflater + .decompress_vec( + &data[in_before as usize..], + &mut out, + FlushDecompress::Finish, + ) + .map_err(|e| format!("deflate: {e}"))?; + if out.len() > limit { + return Err("deflate: output exceeds size limit".into()); + } + match status { + Status::StreamEnd => return Ok(out), + Status::Ok | Status::BufError if out.len() == out.capacity() => { + let grow = out.capacity().min(max_capacity - out.capacity()).max(1); + out.try_reserve_exact(grow) + .map_err(|e| format!("deflate: cannot allocate output: {e}"))?; + } + Status::Ok | Status::BufError => { + if inflater.total_in() as usize >= data.len() + || (inflater.total_in(), inflater.total_out()) == (in_before, out_before) + { + return Err("deflate: truncated stream".into()); + } + } + } + } +} + +/// Compress data using flate2 (zlib-ng, zlib-rs or miniz_oxide; see module docs). pub(crate) fn flate2_compress(data: &[u8], level: u32) -> Result, String> { - use std::io::Write; - let mut encoder = flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::new(level)); - encoder.write_all(data).map_err(|e| e.to_string())?; - encoder.finish().map_err(|e| e.to_string()) + use flate2::{Compress, Compression, FlushCompress, Status}; + + // zlib's compressBound, plus the zlib header and trailer. + let bound = data.len() + (data.len() >> 12) + (data.len() >> 14) + (data.len() >> 25) + 13 + 6; + let mut out = Vec::new(); + out.try_reserve_exact(bound) + .map_err(|e| format!("deflate: cannot allocate output: {e}"))?; + + let mut deflater = Compress::new(Compression::new(level), true); + loop { + let (in_before, out_before) = (deflater.total_in(), deflater.total_out()); + let status = deflater + .compress_vec(&data[in_before as usize..], &mut out, FlushCompress::Finish) + .map_err(|e| format!("deflate: {e}"))?; + match status { + Status::StreamEnd => return Ok(out), + Status::Ok | Status::BufError if out.len() == out.capacity() => out + .try_reserve(out.capacity().max(4096)) + .map_err(|e| format!("deflate: cannot allocate output: {e}"))?, + Status::Ok | Status::BufError => { + if (deflater.total_in(), deflater.total_out()) == (in_before, out_before) { + return Err("deflate: encoder made no progress".into()); + } + } + } + } } // --------------------------------------------------------------------------- @@ -312,7 +365,7 @@ pub(crate) fn flate2_compress(data: &[u8], level: u32) -> Result, String /// /// Selection order: /// 1. Apple Compression Framework (macOS + `apple-compression` feature) -/// 2. flate2 (zlib-ng with `fast-deflate`, otherwise miniz_oxide) +/// 2. flate2 (zlib-ng with `fast-deflate`, else zlib-rs, else miniz_oxide) /// /// When `output_hint` > 0, pre-allocates the output buffer for zero-copy /// decompression (avoids reallocation). @@ -344,7 +397,7 @@ pub fn decompress(data: &[u8], output_hint: usize) -> Result, String> { /// /// Selection order: /// 1. Apple Compression Framework (macOS + `apple-compression` feature) -/// 2. flate2 (zlib-ng with `fast-deflate`, otherwise miniz_oxide) +/// 2. flate2 (zlib-ng with `fast-deflate`, else zlib-rs, else miniz_oxide) pub fn compress(data: &[u8], level: u32) -> Result, String> { #[cfg(all(target_os = "macos", feature = "apple-compression"))] { @@ -377,9 +430,19 @@ pub fn active_backend() -> &'static str { { "zlib-ng" } + // flate2 prefers a C zlib over zlib-rs when both are enabled. + #[cfg(all( + not(all(target_os = "macos", feature = "apple-compression")), + not(feature = "fast-deflate"), + feature = "zlib-rs" + ))] + { + "zlib-rs" + } #[cfg(not(any( all(target_os = "macos", feature = "apple-compression"), - feature = "fast-deflate" + feature = "fast-deflate", + feature = "zlib-rs" )))] { "miniz_oxide" @@ -436,7 +499,7 @@ mod tests { fn backend_name_is_set() { let name = active_backend(); assert!( - ["miniz_oxide", "zlib-ng", "apple-compression"].contains(&name), + ["miniz_oxide", "zlib-rs", "zlib-ng", "apple-compression"].contains(&name), "unexpected backend: {name}" ); } diff --git a/crates/clawhdf5-filters/src/lib.rs b/crates/clawhdf5-filters/src/lib.rs index ae67589..d9d18eb 100644 --- a/crates/clawhdf5-filters/src/lib.rs +++ b/crates/clawhdf5-filters/src/lib.rs @@ -2,12 +2,14 @@ //! //! Provides deflate (zlib) decompression/compression with multiple backend options: //! -//! - **Default**: `miniz_oxide` (pure Rust, no C dependencies) -//! - **`fast-deflate` feature**: `zlib-ng` via flate2 (~2-3x faster, matches C HDF5) +//! - **Default (`zlib-rs` feature)**: `zlib-rs` via flate2 (pure Rust, no C +//! dependencies) +//! - **`fast-deflate` feature**: `zlib-ng` via flate2 (C, built with cmake) //! - **`apple-compression` feature**: Apple Compression Framework on macOS //! (hardware-accelerated on Apple Silicon) +//! - With none of the above: `miniz_oxide` (pure Rust, slower) //! -//! Backend priority: apple-compression > zlib-ng > miniz_oxide. +//! Backend priority: apple-compression > zlib-ng > zlib-rs > miniz_oxide. pub mod fast_deflate; @@ -115,7 +117,7 @@ mod tests { fn backend_reports_name() { let name = deflate_backend(); assert!( - ["miniz_oxide", "zlib-ng", "apple-compression"].contains(&name), + ["miniz_oxide", "zlib-rs", "zlib-ng", "apple-compression"].contains(&name), "unexpected backend: {name}" ); } diff --git a/crates/clawhdf5-format/Cargo.toml b/crates/clawhdf5-format/Cargo.toml index 7263497..fe38392 100644 --- a/crates/clawhdf5-format/Cargo.toml +++ b/crates/clawhdf5-format/Cargo.toml @@ -32,7 +32,10 @@ name = "bench" harness = false [features] -default = ["std", "checksum", "deflate", "provenance", "fast-deflate", "system-zlib-decompress"] +# Deflate backend: `zlib-rs` (pure Rust) by default. `fast-deflate` selects +# zlib-ng instead (C, built with cmake); flate2 prefers a C zlib whenever one +# is enabled, so turning it on anywhere in the build overrides the default. +default = ["std", "checksum", "deflate", "provenance", "zlib-rs", "system-zlib-decompress"] std = [] checksum = [] deflate = ["flate2"] @@ -42,7 +45,10 @@ fast-checksum = ["crc32fast"] fast-deflate = ["flate2/zlib-ng"] system-zlib = ["flate2/zlib-default"] system-zlib-decompress = [] -zlib-rs = ["flate2/zlib-rs"] +# `runtime_detection` gives zlib-rs `std`, which it needs to detect and use +# SIMD at runtime. flate2 enables it by default, but we build flate2 with +# default-features = false, and without it zlib-rs inflates 3.5x slower. +zlib-rs = ["flate2/zlib-rs", "flate2/runtime_detection"] lz4 = ["lz4_flex"] zstd = ["dep:zstd"] blake3_hash = ["blake3"] diff --git a/crates/clawhdf5-format/src/filters.rs b/crates/clawhdf5-format/src/filters.rs index 9979d3f..c10cea9 100644 --- a/crates/clawhdf5-format/src/filters.rs +++ b/crates/clawhdf5-format/src/filters.rs @@ -629,21 +629,70 @@ fn deflate_decompress(data: &[u8], expected_bytes: usize) -> Result, For // Fall through to flate2 on error } - use std::io::Read; - let decoder = flate2::read::ZlibDecoder::new(data); - let mut result = Vec::with_capacity(limit.min(1 << 20)); - // Read one byte past the limit so an over-size stream is distinguishable + // A chunk's decompressed size is known, so allocate it once; without one, + // start from a multiple of the input and grow. + let size_hint = if expected_bytes != 0 { + expected_bytes + } else { + data.len().saturating_mul(4).min(1 << 20) + }; + inflate_bounded(data, size_hint, limit).map_err(FormatError::DecompressionError) +} + +/// Inflate a zlib stream into a buffer sized up front, handing the decoder the +/// whole input at once. +/// +/// `flate2::read::ZlibDecoder` feeds its input through a 32 KiB buffer and +/// grows the output as it goes; on single chunks that cost zlib-rs up to 3.7x +/// against zlib-ng (`BENCHMARKS.md`, "Deflate backend"). Output beyond `limit` +/// is an error, as is a stream that ends before its end-of-stream marker (the +/// streaming reader returned the bytes it had and no error). +#[cfg(feature = "deflate")] +pub(crate) fn inflate_bounded( + data: &[u8], + size_hint: usize, + limit: usize, +) -> Result, String> { + use flate2::{Decompress, FlushDecompress, Status}; + + // One byte of headroom past the limit distinguishes an over-size stream // from one that legitimately ends exactly at the limit. - decoder - .take(limit as u64 + 1) - .read_to_end(&mut result) - .map_err(|e| FormatError::DecompressionError(e.to_string()))?; - if result.len() > limit { - return Err(FormatError::DecompressionError( - "deflate: output exceeds size limit".into(), - )); + let max_capacity = limit.saturating_add(1); + let mut out = Vec::new(); + out.try_reserve_exact(size_hint.clamp(1, max_capacity)) + .map_err(|e| format!("deflate: cannot allocate output: {e}"))?; + + let mut inflater = Decompress::new(true); + loop { + let (in_before, out_before) = (inflater.total_in(), inflater.total_out()); + let status = inflater + .decompress_vec( + &data[in_before as usize..], + &mut out, + FlushDecompress::Finish, + ) + .map_err(|e| format!("deflate: {e}"))?; + if out.len() > limit { + return Err("deflate: output exceeds size limit".into()); + } + match status { + Status::StreamEnd => return Ok(out), + Status::Ok | Status::BufError if out.len() == out.capacity() => { + // Out of room: double, up to the limit. + let grow = out.capacity().min(max_capacity - out.capacity()).max(1); + out.try_reserve_exact(grow) + .map_err(|e| format!("deflate: cannot allocate output: {e}"))?; + } + Status::Ok | Status::BufError => { + // Room left, so the decoder stopped for want of input. + if inflater.total_in() as usize >= data.len() + || (inflater.total_in(), inflater.total_out()) == (in_before, out_before) + { + return Err("deflate: truncated stream".into()); + } + } + } } - Ok(result) } /// Direct FFI to Apple's system libz for fast decompression. @@ -722,14 +771,41 @@ fn deflate_decompress(_data: &[u8], _expected_bytes: usize) -> Result, F /// Compress data with zlib. #[cfg(feature = "deflate")] fn deflate_compress(data: &[u8], level: u32) -> Result, FormatError> { - use std::io::Write; - let mut encoder = flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::new(level)); - encoder - .write_all(data) - .map_err(|e| FormatError::CompressionError(e.to_string()))?; - encoder - .finish() - .map_err(|e| FormatError::CompressionError(e.to_string())) + deflate_bounded(data, level).map_err(FormatError::CompressionError) +} + +/// Deflate `data` into a zlib stream in one pass, into a buffer sized for the +/// worst case up front (the same reasoning as [`inflate_bounded`]). +#[cfg(feature = "deflate")] +pub(crate) fn deflate_bounded(data: &[u8], level: u32) -> Result, String> { + use flate2::{Compress, Compression, FlushCompress, Status}; + + // zlib's compressBound, plus the zlib header and trailer. + let bound = data.len() + (data.len() >> 12) + (data.len() >> 14) + (data.len() >> 25) + 13 + 6; + let mut out = Vec::new(); + out.try_reserve_exact(bound) + .map_err(|e| format!("deflate: cannot allocate output: {e}"))?; + + let mut deflater = Compress::new(Compression::new(level), true); + loop { + let (in_before, out_before) = (deflater.total_in(), deflater.total_out()); + let status = deflater + .compress_vec(&data[in_before as usize..], &mut out, FlushCompress::Finish) + .map_err(|e| format!("deflate: {e}"))?; + match status { + Status::StreamEnd => return Ok(out), + // The bound should make running out of room unreachable; grow + // rather than fail if it happens. + Status::Ok | Status::BufError if out.len() == out.capacity() => out + .try_reserve(out.capacity().max(4096)) + .map_err(|e| format!("deflate: cannot allocate output: {e}"))?, + Status::Ok | Status::BufError => { + if (deflater.total_in(), deflater.total_out()) == (in_before, out_before) { + return Err("deflate: encoder made no progress".into()); + } + } + } + } } #[cfg(not(feature = "deflate"))] @@ -1833,6 +1909,74 @@ mod tests { assert!(deflate_decompress(&compressed, 64).is_err()); } + #[cfg(feature = "deflate")] + fn noisy_bytes(n: usize) -> Vec { + // Compressible but not trivially so. + (0..n) + .map(|i| ((i as f64 * 0.01).sin() * 127.0 + 128.0) as u8 ^ (i as u8 & 3)) + .collect() + } + + #[test] + #[cfg(feature = "deflate")] + fn deflate_decompress_accepts_output_exactly_at_chunk_size() { + let data = noisy_bytes(100_000); + let compressed = deflate_compress(&data, 6).unwrap(); + assert_eq!(deflate_decompress(&compressed, data.len()).unwrap(), data); + // One byte short of the real size is over the limit. + assert!(deflate_decompress(&compressed, data.len() - 1).is_err()); + } + + #[test] + #[cfg(feature = "deflate")] + fn deflate_decompress_without_size_grows_the_buffer() { + // No chunk size: the output starts at 4x the input and has to grow. + let data = vec![7u8; 3 * 1024 * 1024]; + let compressed = deflate_compress(&data, 6).unwrap(); + assert!(compressed.len() * 4 < data.len()); + assert_eq!(deflate_decompress(&compressed, 0).unwrap(), data); + } + + #[test] + #[cfg(feature = "deflate")] + fn deflate_decompress_rejects_truncated_stream() { + // The streaming reader this replaced returned the bytes it had and no + // error, so a truncated chunk read back short. + let data = noisy_bytes(100_000); + let compressed = deflate_compress(&data, 6).unwrap(); + for cut in [compressed.len() - 1, compressed.len() / 2, 3] { + assert!( + deflate_decompress(&compressed[..cut], data.len()).is_err(), + "truncated to {cut} of {} bytes", + compressed.len() + ); + } + } + + #[test] + #[cfg(feature = "deflate")] + fn deflate_compress_roundtrips_incompressible_data() { + // Random-looking input compresses to slightly more than it started + // as; the output must still fit the pre-sized buffer (or grow). + let mut x = 0x9E37_79B9_7F4A_7C15u64; + let data: Vec = (0..200_000) + .map(|_| { + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + x as u8 + }) + .collect(); + for level in [0, 1, 6, 9] { + let compressed = deflate_compress(&data, level).unwrap(); + assert_eq!(deflate_decompress(&compressed, data.len()).unwrap(), data); + } + assert_eq!( + deflate_decompress(&deflate_compress(&[], 6).unwrap(), 0).unwrap(), + Vec::::new() + ); + } + #[test] #[cfg(feature = "zstd")] fn zstd_decompress_rejects_output_exceeding_chunk_size() { diff --git a/crates/clawhdf5/Cargo.toml b/crates/clawhdf5/Cargo.toml index ee5326f..40dbec2 100644 --- a/crates/clawhdf5/Cargo.toml +++ b/crates/clawhdf5/Cargo.toml @@ -30,9 +30,10 @@ name = "parallel_bench" harness = false [features] -default = ["mmap", "fast-deflate", "provenance"] +default = ["mmap", "provenance"] mmap = ["clawhdf5-io/mmap"] parallel = ["clawhdf5-format/parallel", "rayon"] +# zlib-ng (C, needs cmake) instead of the default pure-Rust zlib-rs. fast-deflate = ["clawhdf5-format/fast-deflate"] apple-compression = [] zstd = ["clawhdf5-format/zstd"] diff --git a/scripts/ci-test.sh b/scripts/ci-test.sh index f04dd09..4df9e33 100755 --- a/scripts/ci-test.sh +++ b/scripts/ci-test.sh @@ -77,6 +77,35 @@ run_step "cargo clippy (ann parallel)" cargo clippy \ --features parallel \ -- -D warnings +# zlib-ng is opt-in (`fast-deflate`; the default is pure-Rust zlib-rs), so +# nothing above builds it. Keep it compiling and passing. +run_step "cargo clippy (fast-deflate / zlib-ng)" cargo clippy \ + -p clawhdf5-format -p clawhdf5-filters -p clawhdf5 \ + --all-targets \ + --features clawhdf5-format/fast-deflate,clawhdf5-filters/fast-deflate \ + -- -D warnings + +# The README promises that the core crates build no C by default. Hold it to +# that: fail if a crate that compiles C (a *-sys crate, cc or cmake) enters the +# default dependency tree of any of them. clawhdf5-migrate (bundled SQLite), +# clawhdf5-napi (Node) and clawhdf5-gpu (graphics drivers) are exempt. +no_c_in_default_build() { + local crate found=0 + for crate in clawhdf5-format clawhdf5-io clawhdf5-filters clawhdf5 \ + clawhdf5-agent clawhdf5-ann clawhdf5-accel clawhdf5-netcdf4 clawhdf5-cli; do + local c_deps + c_deps=$(cargo tree -q -p "$crate" -e normal,build --prefix none \ + | grep -E '^([a-z0-9_-]+-sys|cc|cmake) v' | sort -u) + if [ -n "$c_deps" ]; then + echo "$crate pulls in C by default:" + echo "$c_deps" | sed 's/^/ /' + found=1 + fi + done + return $found +} +run_step "no C in the default build (core crates)" no_c_in_default_build + # 4. Tests (exclude clawhdf5-py) run_step "cargo test" cargo test \ --workspace \ @@ -90,6 +119,10 @@ run_step "cargo test (ann parallel)" cargo test \ -p clawhdf5-ann \ --features parallel +run_step "cargo test (fast-deflate / zlib-ng)" cargo test \ + -p clawhdf5-format -p clawhdf5-filters -p clawhdf5 \ + --features clawhdf5-format/fast-deflate,clawhdf5-filters/fast-deflate + # 5. Python interop suites. The h5py writer tests are #[ignore]d so a plain # `cargo test` stays hermetic; run them explicitly here. # On a PEP 668 "externally managed" system h5py can only live in a