osobhandClaude Opus 5.5
d0db83812b
The setting was persisted in /meta and otherwise ignored: embeddings
were always written as f32. It now does what it says.
clawhdf5-format:
- `DatasetBuilder::with_f16_data` writes IEEE binary16 (numpy float16),
rounding to nearest-even, and `make_f16_type`.
- `clawhdf5_format::float16` holds the f32 <-> f16 conversions, the one
implementation the writer, the reader and the agent all use. Checked
against the `half` crate on 16.7M f32 values and round-trips all 65536
half values; the h5py interop tests confirm the rounding matches
numpy's bit for bit (4020 values incl. ties, subnormals, overflow).
- Reading little-endian float16 as f32 has a fast path.
clawhdf5-agent:
- A float16 store writes /memory/embeddings as half precision, and
`MemoryCache::half_precision` rounds each embedding as it enters the
cache (save, update, WAL replay, and on load of a store still f32 on
disk), so memory and file agree bit for bit and a store searches the
same before and after a reopen (tested).
- Values beyond +-65504 are refused with the new
`MemoryError::InvalidEntry` rather than stored as infinity, on every
save path; batches are all or nothing, and a rejected ephemeral entry
stays in the ephemeral tier. Breaking for exhaustive matches.
- CLI: `create --float16`. Off by default.
Measured on tank, 384-dim, six runs alternating order, medians
(search_harness --float16-study --full): at 100K the file goes from
154.0 to 80.8 MiB (-48%), checkpoint 752 -> 512 ms, open 300 -> 252 ms;
vector recall@10 against an exact scan and hybrid_search latency do not
change. At 10K open is 3 ms slower. Also a test that h5py opens a whole
agent store, f32 and float16, and decodes every dataset.
Docs: README, BENCHMARKS.md ("float16 embedding storage"), CHANGELOG
(including the h5py interop fixes in the previous commit), CLAUDE.md.
Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
64 lines
2.2 KiB
TOML
64 lines
2.2 KiB
TOML
[package]
|
|
name = "clawhdf5-format"
|
|
version = "2.7.0"
|
|
edition = "2024"
|
|
rust-version.workspace = true
|
|
description = "Pure-Rust HDF5 binary format parsing and writing — no C dependencies"
|
|
license = "MIT"
|
|
repository = "https://git.redclaw.dev/quantumclaw/clawhdf5"
|
|
readme = "README.md"
|
|
keywords = ["hdf5", "science", "data", "binary", "no-std"]
|
|
categories = ["parser-implementations", "science", "encoding", "no-std"]
|
|
|
|
[dependencies]
|
|
byteorder = { version = "1", default-features = false }
|
|
portable-atomic = { version = "1" }
|
|
flate2 = { version = "1", default-features = false, features = ["rust_backend"], optional = true }
|
|
sha2 = { version = "0.10", default-features = false, optional = true }
|
|
rayon = { version = "1", optional = true }
|
|
crc32fast = { version = "1", optional = true }
|
|
lz4_flex = { version = "0.11", optional = true }
|
|
zstd = { version = "0.13", optional = true }
|
|
blake3 = { version = "1", optional = true }
|
|
libaec-sys = { path = "../libaec-sys", version = "0.1", optional = true }
|
|
pco = { version = "1.0", optional = true }
|
|
|
|
[dev-dependencies]
|
|
half = { workspace = true }
|
|
serde_json = "1"
|
|
criterion = { workspace = true }
|
|
clawhdf5-derive = { path = "../clawhdf5-derive", version = "2.7.0" }
|
|
|
|
[[bench]]
|
|
name = "bench"
|
|
harness = false
|
|
|
|
[features]
|
|
# Deflate backend: `zlib-rs` (pure Rust) by default. `fast-deflate` selects
|
|
# zlib-ng instead (C, built with cmake); flate2 prefers a C zlib whenever one
|
|
# is enabled, so turning it on anywhere in the build overrides the default.
|
|
default = ["std", "checksum", "deflate", "provenance", "zlib-rs", "system-zlib-decompress"]
|
|
std = []
|
|
checksum = []
|
|
deflate = ["flate2"]
|
|
provenance = ["sha2"]
|
|
parallel = ["rayon", "std"]
|
|
fast-checksum = ["crc32fast"]
|
|
fast-deflate = ["flate2/zlib-ng"]
|
|
system-zlib = ["flate2/zlib-default"]
|
|
system-zlib-decompress = []
|
|
# `runtime_detection` gives zlib-rs `std`, which it needs to detect and use
|
|
# SIMD at runtime. flate2 enables it by default, but we build flate2 with
|
|
# default-features = false, and without it zlib-rs inflates 3.5x slower.
|
|
zlib-rs = ["flate2/zlib-rs", "flate2/runtime_detection"]
|
|
lz4 = ["lz4_flex"]
|
|
zstd = ["dep:zstd"]
|
|
blake3_hash = ["blake3"]
|
|
szip = ["libaec-sys"]
|
|
pcodec = ["dep:pco"]
|
|
|
|
[[bench]]
|
|
name = "parallel_decompress_bench"
|
|
harness = false
|
|
required-features = ["parallel"]
|