perf(agent): persist the vector index graph; incremental catch-up

open() marked the HNSW index dirty, so the first search of every session
rebuilt it from scratch — 36 s at 100K records with the (better, slower)
heuristic build. First query after open is now 1.7 / 15 / 159 ms at
1K / 10K / 100K; what remains is the one-off keyword index build.

- clawhdf5-ann: HnswIndex::graph_to_bytes / from_graph_bytes serialize the
  graph only (levels, tombstones, adjacency as u32, CRC32). The existing HDF5
  serializer embeds a full copy of every vector, which would double a store
  that already holds them. Loading validates everything — counts, levels vs
  layer count, connection limits, every neighbour id and the layer it must
  exist on — so a damaged graph, or a hostile one with a valid checksum, is an
  error rather than an out-of-bounds walk during search.
- clawhdf5-agent: each checkpoint writes the graph to <store>.h5.ann (synced,
  atomic, before the .h5) and records a fresh generation id in /meta. open()
  loads the sidecar only if its generation matches that checkpoint; missing,
  stale, damaged or mismatched sidecars are ignored and the index rebuilt.
  Records appended through WAL replay join the loaded index incrementally; a
  replayed Update or Tombstone invalidates it. snapshot() copies it. Only an
  index that exactly mirrors the cache is saved; otherwise a stale sidecar is
  removed.
- ensure_hnsw_fresh inserts records appended since the last sync instead of
  rebuilding, so save_batch no longer marks the whole index dirty.
- CheckpointMeta { wal_applied, ann_generation } with *_with_meta build/write/
  read functions; the *_with_mark ones delegate.
- Harness reports the one-off cold index build separately from the first query
  after a reopen.

Co-Authored-By: Claude Fable 5.1 <[email protected]>

crates/clawhdf5-agent/src/storage.rs

@@ -34,7 +34,23 @@ pub fn write_to_disk_with_mark(
     knowledge: &KnowledgeCache,
     wal_applied: Option<WalMark>,
 ) -> Result<(), MemoryError> {
-    let bytes = schema::build_hdf5_file_with_mark(config, cache, sessions, knowledge, wal_applied)?;
+    let meta = schema::CheckpointMeta {
+        wal_applied,
+        ann_generation: None,
+    };
+    write_to_disk_with_meta(path, config, cache, sessions, knowledge, &meta)
+}
+
+/// [`write_to_disk`] with full checkpoint bookkeeping.
+pub fn write_to_disk_with_meta(
+    path: &Path,
+    config: &MemoryConfig,
+    cache: &MemoryCache,
+    sessions: &SessionCache,
+    knowledge: &KnowledgeCache,
+    checkpoint: &schema::CheckpointMeta,
+) -> Result<(), MemoryError> {
+    let bytes = schema::build_hdf5_file_with_meta(config, cache, sessions, knowledge, checkpoint)?;
 
     if bytes.is_empty() {
         return Err(MemoryError::Hdf5("build_hdf5_file produced 0 bytes".into()));
@@ -47,7 +63,7 @@ pub fn write_to_disk_with_mark(
 }
 
 /// Write `bytes` to `path` and flush them to stable storage.
-fn write_synced(path: &Path, bytes: &[u8]) -> Result<(), MemoryError> {
+pub(crate) fn write_synced(path: &Path, bytes: &[u8]) -> Result<(), MemoryError> {
     use std::io::Write;
     let mut f = std::fs::File::create(path).map_err(MemoryError::Io)?;
     f.write_all(bytes).map_err(MemoryError::Io)?;
@@ -62,7 +78,7 @@ fn write_synced(path: &Path, bytes: &[u8]) -> Result<(), MemoryError> {
 /// This is per-checkpoint/snapshot cost only (each is already a full file
 /// write). Individual WAL appends are deliberately not synced — see the
 /// durability notes in the crate docs.
-fn rename_synced(from: &Path, to: &Path) -> Result<(), MemoryError> {
+pub(crate) fn rename_synced(from: &Path, to: &Path) -> Result<(), MemoryError> {
     std::fs::rename(from, to).map_err(MemoryError::Io)?;
     #[cfg(unix)]
     if let Some(dir) = to.parent() {
@@ -113,6 +129,20 @@ pub fn read_from_disk_with_mark(path: &Path) -> Result<(StoreState, Option<WalMa
     Ok(((config, cache, sessions, knowledge), wal_applied))
 }
 
+/// [`read_from_disk`], plus all checkpoint bookkeeping.
+pub fn read_from_disk_with_meta(
+    path: &Path,
+) -> Result<(StoreState, schema::CheckpointMeta), MemoryError> {
+    let mmap = clawhdf5_io::MmapReader::open(path).map_err(MemoryError::Io)?;
+    mmap.advise_willneed(0, mmap.len());
+    let file = clawhdf5::File::from_bytes(mmap.as_bytes().to_vec())
+        .map_err(|e| MemoryError::Hdf5(format!("cannot open {}: {e}", path.display())))?;
+    let (mut config, cache, sessions, knowledge) = schema::validate_and_load(&file)?;
+    config.path = path.to_path_buf();
+    let meta = schema::read_checkpoint_meta(&file);
+    Ok(((config, cache, sessions, knowledge), meta))
+}
+
 /// Copy an HDF5 file atomically to a destination.
 pub fn snapshot_file(src: &Path, dest: &Path) -> Result<std::path::PathBuf, MemoryError> {
     let dest_file = if dest.is_dir() {