feat(agent): MemoryConfig::float16 stores half-precision embeddings
The setting was persisted in /meta and otherwise ignored: embeddings
were always written as f32. It now does what it says.
clawhdf5-format:
- `DatasetBuilder::with_f16_data` writes IEEE binary16 (numpy float16),
rounding to nearest-even, and `make_f16_type`.
- `clawhdf5_format::float16` holds the f32 <-> f16 conversions, the one
implementation the writer, the reader and the agent all use. Checked
against the `half` crate on 16.7M f32 values and round-trips all 65536
half values; the h5py interop tests confirm the rounding matches
numpy's bit for bit (4020 values incl. ties, subnormals, overflow).
- Reading little-endian float16 as f32 has a fast path.
clawhdf5-agent:
- A float16 store writes /memory/embeddings as half precision, and
`MemoryCache::half_precision` rounds each embedding as it enters the
cache (save, update, WAL replay, and on load of a store still f32 on
disk), so memory and file agree bit for bit and a store searches the
same before and after a reopen (tested).
- Values beyond +-65504 are refused with the new
`MemoryError::InvalidEntry` rather than stored as infinity, on every
save path; batches are all or nothing, and a rejected ephemeral entry
stays in the ephemeral tier. Breaking for exhaustive matches.
- CLI: `create --float16`. Off by default.
Measured on tank, 384-dim, six runs alternating order, medians
(search_harness --float16-study --full): at 100K the file goes from
154.0 to 80.8 MiB (-48%), checkpoint 752 -> 512 ms, open 300 -> 252 ms;
vector recall@10 against an exact scan and hybrid_search latency do not
change. At 10K open is 3 ms slower. Also a test that h5py opens a whole
agent store, f32 and float16, and decodes every dataset.
Docs: README, BENCHMARKS.md ("float16 embedding storage"), CHANGELOG
(including the h5py interop fixes in the previous commit), CLAUDE.md.
Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
@@ -159,20 +159,27 @@ fn build_memory_group(
|
||||
// chunks: fixed-length string array
|
||||
write_string_dataset(&mut group, "chunks", &cache.chunks);
|
||||
|
||||
// embeddings: f32 [N x D]
|
||||
// embeddings: [N x D], f32 — or IEEE half precision for a `float16`
|
||||
// store. The cache already holds half-rounded values then, so this
|
||||
// conversion is exact and a reopened store sees the same numbers.
|
||||
let n = cache.embeddings.len() as u64;
|
||||
let d = cache.embedding_dim as u64;
|
||||
let flat = cache.flat_embeddings();
|
||||
{
|
||||
let ds = group
|
||||
.create_dataset("embeddings")
|
||||
.with_f32_data(flat)
|
||||
.with_shape(&[n, d]);
|
||||
let ds = group.create_dataset("embeddings");
|
||||
let elem_bytes: u64 = if config.float16 {
|
||||
ds.with_f16_data(flat);
|
||||
2
|
||||
} else {
|
||||
ds.with_f32_data(flat);
|
||||
4
|
||||
};
|
||||
ds.with_shape(&[n, d]);
|
||||
|
||||
// Chunk size tuning: target ~256KB per chunk for optimal I/O
|
||||
if n > 0 && d > 0 {
|
||||
let target_chunk_bytes: u64 = 256 * 1024;
|
||||
let rows_per_chunk = (target_chunk_bytes / (d * 4)).max(1).min(n);
|
||||
let rows_per_chunk = (target_chunk_bytes / (d * elem_bytes)).max(1).min(n);
|
||||
ds.with_chunks(&[rows_per_chunk, d]);
|
||||
|
||||
// Compression. Shuffle is applied automatically (auto-shuffle
|
||||
@@ -513,7 +520,16 @@ pub fn validate_and_load(
|
||||
};
|
||||
|
||||
// Load /memory group
|
||||
let memory_cache = load_memory_group(file, embedding_dim)?;
|
||||
let mut memory_cache = load_memory_group(file, embedding_dim)?;
|
||||
// A float16 store's cache holds half-rounded embeddings. Embeddings read
|
||||
// from an f16 dataset already are; a float16 store whose last checkpoint
|
||||
// predates half-precision storage is still f32 on disk and is rounded
|
||||
// here.
|
||||
if config.float16 && embeddings_are_f16(file) {
|
||||
memory_cache.half_precision = true;
|
||||
} else {
|
||||
memory_cache.set_half_precision(config.float16);
|
||||
}
|
||||
|
||||
// Load /sessions group
|
||||
let session_cache = load_sessions_group(file)?;
|
||||
@@ -756,6 +772,13 @@ fn read_string_dataset_from_group(
|
||||
.map_err(|e| MemoryError::Hdf5(format!("cannot read strings from {name}: {e}")))
|
||||
}
|
||||
|
||||
/// Whether `/memory/embeddings` is stored as IEEE half precision.
|
||||
fn embeddings_are_f16(file: &clawhdf5::File) -> bool {
|
||||
file.dataset("memory/embeddings")
|
||||
.and_then(|ds| ds.dtype())
|
||||
.is_ok_and(|dt| matches!(dt, clawhdf5::DType::Other(ref s) if s == "float16"))
|
||||
}
|
||||
|
||||
fn read_f32_dataset(group: &clawhdf5::Group<'_>, name: &str) -> Result<Vec<f32>, MemoryError> {
|
||||
let ds = group
|
||||
.dataset(name)
|
||||
|
||||
Reference in New Issue
Block a user