perf(agent): persist the vector index graph; incremental catch-up

open() marked the HNSW index dirty, so the first search of every session
rebuilt it from scratch — 36 s at 100K records with the (better, slower)
heuristic build. First query after open is now 1.7 / 15 / 159 ms at
1K / 10K / 100K; what remains is the one-off keyword index build.

- clawhdf5-ann: HnswIndex::graph_to_bytes / from_graph_bytes serialize the
  graph only (levels, tombstones, adjacency as u32, CRC32). The existing HDF5
  serializer embeds a full copy of every vector, which would double a store
  that already holds them. Loading validates everything — counts, levels vs
  layer count, connection limits, every neighbour id and the layer it must
  exist on — so a damaged graph, or a hostile one with a valid checksum, is an
  error rather than an out-of-bounds walk during search.
- clawhdf5-agent: each checkpoint writes the graph to <store>.h5.ann (synced,
  atomic, before the .h5) and records a fresh generation id in /meta. open()
  loads the sidecar only if its generation matches that checkpoint; missing,
  stale, damaged or mismatched sidecars are ignored and the index rebuilt.
  Records appended through WAL replay join the loaded index incrementally; a
  replayed Update or Tombstone invalidates it. snapshot() copies it. Only an
  index that exactly mirrors the cache is saved; otherwise a stale sidecar is
  removed.
- ensure_hnsw_fresh inserts records appended since the last sync instead of
  rebuilding, so save_batch no longer marks the whole index dirty.
- CheckpointMeta { wal_applied, ann_generation } with *_with_meta build/write/
  read functions; the *_with_mark ones delegate.
- Harness reports the one-off cold index build separately from the first query
  after a reopen.

Co-Authored-By: Claude Fable 5.1 <[email protected]>

crates/clawhdf5-ann/src/hnsw.rs

@@ -134,6 +134,9 @@ impl Ord for FarCandidate {
     }
 }
 
+/// Magic for [`HnswIndex::graph_to_bytes`].
+const GRAPH_MAGIC: &[u8; 4] = b"CHG1";
+
 /// On-disk format version for the serialized HNSW index.
 ///
 /// - Version 1: original layout (`vectors`, `graph_layer_*`, `config`), no
@@ -690,6 +693,160 @@ impl HnswIndex {
         })
     }
 
+    /// Serialize the **graph only** — levels, tombstones and adjacency, not the
+    /// vectors — for a caller that already stores the vectors elsewhere (the
+    /// agent's record cache). [`HnswIndex::to_hdf5_bytes`] writes a complete,
+    /// self-contained index including a full copy of every vector, which would
+    /// double such a store's size. Reattach with
+    /// [`HnswIndex::from_graph_bytes`].
+    ///
+    /// Layout (little endian): magic `CHG1`, then u32 fields `n`, `m`,
+    /// `m_max0`, `ef_construction`, `entry_point`, `num_layers`, `metric`;
+    /// `n` level bytes; `n` tombstone bytes; per layer, per node that exists on
+    /// that layer: u32 neighbour count + u32 ids; trailing CRC32 of all of it.
+    pub fn graph_to_bytes(&self) -> Vec<u8> {
+        let n = self.vectors.len();
+        let mut out = Vec::with_capacity(32 + n * 2 + n * self.m_max0 * 4);
+        out.extend_from_slice(GRAPH_MAGIC);
+        for field in [
+            n,
+            self.m,
+            self.m_max0,
+            self.ef_construction,
+            self.entry_point,
+            self.graph.len(),
+            match self.metric {
+                DistanceMetric::L2 => 0,
+                DistanceMetric::Cosine => 1,
+            },
+        ] {
+            out.extend_from_slice(&(field as u32).to_le_bytes());
+        }
+        out.extend(self.node_levels.iter().map(|&l| l.min(255) as u8));
+        out.extend(self.deleted.iter().map(|&d| u8::from(d)));
+        for (layer, adjacency) in self.graph.iter().enumerate() {
+            for (node, neighbors) in adjacency.iter().enumerate() {
+                if self.node_levels[node] < layer {
+                    continue; // node does not exist on this layer
+                }
+                out.extend_from_slice(&(neighbors.len() as u32).to_le_bytes());
+                for &id in neighbors {
+                    out.extend_from_slice(&(id as u32).to_le_bytes());
+                }
+            }
+        }
+        let crc = clawhdf5_format::checksum::crc32(&out);
+        out.extend_from_slice(&crc.to_le_bytes());
+        out
+    }
+
+    /// Rebuild an index from [`HnswIndex::graph_to_bytes`] output and the
+    /// vectors it was built over (same order). Every structural claim in
+    /// `bytes` is validated — a corrupt or mismatched graph is an error, never
+    /// an index that panics or walks out of bounds during a search.
+    pub fn from_graph_bytes(bytes: &[u8], vectors: Vec<Vec<f32>>) -> Result<Self, FormatError> {
+        let bad = |what: &str| FormatError::SerializationError(format!("HNSW graph: {what}"));
+        let body_len = bytes
+            .len()
+            .checked_sub(4)
+            .filter(|&l| l >= GRAPH_MAGIC.len() + 7 * 4)
+            .ok_or_else(|| bad("truncated"))?;
+        let (body, crc_bytes) = bytes.split_at(body_len);
+        if &body[..4] != GRAPH_MAGIC {
+            return Err(bad("bad magic"));
+        }
+        let stored_crc =
+            u32::from_le_bytes([crc_bytes[0], crc_bytes[1], crc_bytes[2], crc_bytes[3]]);
+        if clawhdf5_format::checksum::crc32(body) != stored_crc {
+            return Err(bad("checksum mismatch"));
+        }
+
+        let mut pos = 4;
+        let next_u32 = |pos: &mut usize| -> Result<usize, FormatError> {
+            let b = body.get(*pos..*pos + 4).ok_or_else(|| bad("truncated"))?;
+            *pos += 4;
+            Ok(u32::from_le_bytes([b[0], b[1], b[2], b[3]]) as usize)
+        };
+        let n = next_u32(&mut pos)?;
+        let m = next_u32(&mut pos)?;
+        let m_max0 = next_u32(&mut pos)?;
+        let ef_construction = next_u32(&mut pos)?;
+        let entry_point = next_u32(&mut pos)?;
+        let num_layers = next_u32(&mut pos)?;
+        let metric = match next_u32(&mut pos)? {
+            0 => DistanceMetric::L2,
+            1 => DistanceMetric::Cosine,
+            _ => return Err(bad("unknown metric")),
+        };
+        if n != vectors.len() {
+            return Err(bad("vector count does not match the graph"));
+        }
+        if n == 0 || entry_point >= n || m < 2 || num_layers == 0 || num_layers > 256 {
+            return Err(bad("invalid header"));
+        }
+        let dim = vectors[0].len();
+        if vectors.iter().any(|v| v.len() != dim) {
+            return Err(bad("vectors have mixed dimensions"));
+        }
+
+        let levels = body.get(pos..pos + n).ok_or_else(|| bad("truncated"))?;
+        pos += n;
+        let node_levels: Vec<usize> = levels.iter().map(|&l| l as usize).collect();
+        if node_levels.iter().any(|&l| l >= num_layers)
+            || node_levels[entry_point] + 1 != num_layers
+        {
+            return Err(bad("levels inconsistent with layer count"));
+        }
+        let deleted: Vec<bool> = body
+            .get(pos..pos + n)
+            .ok_or_else(|| bad("truncated"))?
+            .iter()
+            .map(|&d| d != 0)
+            .collect();
+        pos += n;
+
+        let mut graph: Vec<Vec<Vec<usize>>> = Vec::with_capacity(num_layers);
+        for layer in 0..num_layers {
+            let max_conn = if layer == 0 { m_max0 } else { m };
+            let mut adjacency = vec![Vec::new(); n];
+            for (node, slot) in adjacency.iter_mut().enumerate() {
+                if node_levels[node] < layer {
+                    continue;
+                }
+                let count = next_u32(&mut pos)?;
+                if count > max_conn {
+                    return Err(bad("neighbour list exceeds the connection limit"));
+                }
+                let mut neighbors = Vec::with_capacity(count);
+                for _ in 0..count {
+                    let id = next_u32(&mut pos)?;
+                    // A neighbour must exist, and exist on this layer.
+                    if id >= n || node_levels[id] < layer {
+                        return Err(bad("neighbour id out of range for its layer"));
+                    }
+                    neighbors.push(id);
+                }
+                *slot = neighbors;
+            }
+            graph.push(adjacency);
+        }
+        if pos != body.len() {
+            return Err(bad("trailing bytes"));
+        }
+
+        Ok(Self {
+            vectors,
+            graph,
+            deleted,
+            entry_point,
+            m,
+            m_max0,
+            ef_construction,
+            node_levels,
+            metric,
+        })
+    }
+
     /// Returns the number of vectors in the index.
     pub fn len(&self) -> usize {
         self.vectors.len()
@@ -1147,6 +1304,73 @@ mod tests {
         assert!(recall >= 0.95, "incremental recall@10 = {recall}");
     }
 
+    #[test]
+    fn graph_bytes_round_trip_gives_identical_searches() {
+        let mut vectors = clustered(1260, 16, 12, 9);
+        let queries = vectors.split_off(1200);
+        let mut index = HnswIndex::build_with_metric(&vectors, 8, 40, DistanceMetric::L2);
+        index.mark_deleted(3);
+        index.mark_deleted(700);
+
+        let bytes = index.graph_to_bytes();
+        // The graph is a small fraction of the vectors it indexes... not
+        // necessarily at dim 16, but it must not embed them.
+        assert!(bytes.len() < 1200 * (16 * 2 + 2) * 4);
+        let restored = HnswIndex::from_graph_bytes(&bytes, vectors.clone()).unwrap();
+        assert_eq!(restored.deleted_count(), 2);
+        for q in &queries {
+            assert_eq!(restored.search(q, 10, 50), index.search(q, 10, 50));
+        }
+        // A restored index keeps working incrementally.
+        let mut restored = restored;
+        let id = restored.insert(queries[0].clone());
+        assert_eq!(restored.search(&queries[0], 1, 50)[0].0, id);
+    }
+
+    #[test]
+    fn damaged_or_mismatched_graph_bytes_are_errors() {
+        let vectors = clustered(300, 8, 6, 4);
+        let index = HnswIndex::build_with_metric(&vectors, 6, 30, DistanceMetric::Cosine);
+        let bytes = index.graph_to_bytes();
+
+        // Wrong vector set.
+        assert!(HnswIndex::from_graph_bytes(&bytes, vectors[..299].to_vec()).is_err());
+        // Every truncation.
+        for len in 0..bytes.len() {
+            assert!(
+                HnswIndex::from_graph_bytes(&bytes[..len], vectors.clone()).is_err(),
+                "truncated to {len}"
+            );
+        }
+        // A flipped bit anywhere.
+        for i in (0..bytes.len()).step_by(7) {
+            let mut damaged = bytes.clone();
+            damaged[i] ^= 0x10;
+            assert!(
+                HnswIndex::from_graph_bytes(&damaged, vectors.clone()).is_err(),
+                "bit flip at {i}"
+            );
+        }
+    }
+
+    #[test]
+    fn structurally_invalid_graph_with_a_valid_checksum_is_rejected() {
+        // The CRC only proves the bytes are what was written; a hostile or
+        // buggy writer can checksum nonsense. Out-of-range neighbour ids must
+        // still be caught, or search would index out of bounds.
+        let vectors = clustered(50, 4, 3, 5);
+        let index = HnswIndex::build_with_metric(&vectors, 4, 20, DistanceMetric::L2);
+        let mut bytes = index.graph_to_bytes();
+        let body_len = bytes.len() - 4;
+        // First neighbour id of node 0 on layer 0 sits right after the header,
+        // levels, tombstones and node 0's count.
+        let at = 4 + 7 * 4 + 50 + 50 + 4;
+        bytes[at..at + 4].copy_from_slice(&9999u32.to_le_bytes());
+        let crc = clawhdf5_format::checksum::crc32(&bytes[..body_len]);
+        bytes[body_len..].copy_from_slice(&crc.to_le_bytes());
+        assert!(HnswIndex::from_graph_bytes(&bytes, vectors).is_err());
+    }
+
     #[test]
     fn select_neighbors_prefers_diverse_directions_and_fills_up() {
         // Node at the origin. Three candidates bunched together on the right,