perf(agent): replace BM25 WAND top-k re-sort with a min-heap

top_k_scores.sort_by(...) ran over the full k-sized buffer for every
matching document that beat the running threshold (twice in the full
branch), plus another full sort on first reaching k results —
O(m·k log k) for m matching documents. Replace the Vec<f32> buffer
with a BinaryHeap<Reverse<HeapScore>> min-heap of size k, giving
O(m log k). Existing wand_returns_same_results_as_exhaustive test
confirms results are unchanged.

INT-11

crates/clawhdf5-agent/src/bm25.rs

@@ -8,7 +8,28 @@
 //! - Sorted posting lists by doc_id for cache-friendly access
 //! - Block-Max WAND early termination
 
-use std::collections::HashMap;
+use std::cmp::Reverse;
+use std::collections::{BinaryHeap, HashMap};
+
+/// `f32` wrapper providing a total order (via `total_cmp`) so BM25 scores can
+/// be kept in a `BinaryHeap`. Scores are always finite in practice (no NaN
+/// inputs reach this path), so `total_cmp`'s NaN ordering is never exercised.
+#[derive(Debug, Clone, Copy, PartialEq)]
+struct HeapScore(f32);
+
+impl Eq for HeapScore {}
+
+impl PartialOrd for HeapScore {
+    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
+impl Ord for HeapScore {
+    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+        self.0.total_cmp(&other.0)
+    }
+}
 
 /// Default BM25 term-frequency saturation parameter.
 const DEFAULT_K1: f32 = 1.2;
@@ -97,9 +118,11 @@ impl BM25Index {
 
         let total_max_contribution: f32 = max_tf_score.iter().sum();
 
-        // Threshold for WAND early termination
+        // Threshold for WAND early termination. `top_k_heap` is a min-heap of
+        // size k (worst-of-the-top-k at the head) so it can be maintained in
+        // O(log k) per update instead of re-sorting the whole buffer.
         let mut threshold = 0.0f32;
-        let mut top_k_scores: Vec<f32> = Vec::with_capacity(k);
+        let mut top_k_heap: BinaryHeap<Reverse<HeapScore>> = BinaryHeap::with_capacity(k);
 
         for (term_idx, (_, idf, postings)) in query_terms.iter().enumerate() {
             for &(doc_id, freq) in *postings {
@@ -118,24 +141,17 @@ impl BM25Index {
                 if term_idx == query_terms.len() - 1 {
                     // Last term: check if this doc beats threshold
                     let final_score = *entry;
-                    if final_score > threshold && top_k_scores.len() >= k {
-                        // Update threshold
-                        top_k_scores
-                            .sort_by(|a, b| b.partial_cmp(a).unwrap_or(std::cmp::Ordering::Equal));
-                        if final_score > top_k_scores[k - 1] {
-                            top_k_scores[k - 1] = final_score;
-                            top_k_scores.sort_by(|a, b| {
-                                b.partial_cmp(a).unwrap_or(std::cmp::Ordering::Equal)
-                            });
-                            threshold = top_k_scores[k - 1];
+                    if top_k_heap.len() >= k {
+                        if final_score > threshold {
+                            // Replace the current worst-of-top-k.
+                            top_k_heap.pop();
+                            top_k_heap.push(Reverse(HeapScore(final_score)));
+                            threshold = top_k_heap.peek().map(|Reverse(s)| s.0).unwrap_or(0.0);
                         }
-                    } else if top_k_scores.len() < k {
-                        top_k_scores.push(final_score);
-                        if top_k_scores.len() == k {
-                            top_k_scores.sort_by(|a, b| {
-                                b.partial_cmp(a).unwrap_or(std::cmp::Ordering::Equal)
-                            });
-                            threshold = top_k_scores[k - 1];
+                    } else {
+                        top_k_heap.push(Reverse(HeapScore(final_score)));
+                        if top_k_heap.len() == k {
+                            threshold = top_k_heap.peek().map(|Reverse(s)| s.0).unwrap_or(0.0);
                         }
                     }
                 }

crates/clawhdf5-agent/src/consolidation.rs

@@ -118,7 +118,7 @@ impl ImportanceScorer {
 
     /// Novelty score: 1.0 − max cosine similarity against all existing records.
     /// Returns 1.0 when there are no existing memories.
-    pub fn score_surprise(embedding: &[f32], existing_memories: &[MemoryRecord]) -> f32 {
+    pub fn score_surprise(embedding: &[f32], existing_memories: &[&MemoryRecord]) -> f32 {
         if existing_memories.is_empty() {
             return 1.0;
         }
@@ -209,11 +209,10 @@ impl ConsolidationEngine {
         source: MemorySource,
         now: f64,
     ) -> u64 {
-        let working: Vec<MemoryRecord> = self
+        let working: Vec<&MemoryRecord> = self
             .records
             .iter()
             .filter(|r| r.tier == MemoryTier::Working)
-            .cloned()
             .collect();
 
         let surprise = ImportanceScorer::score_surprise(&embedding, &working);
@@ -281,7 +280,7 @@ impl ConsolidationEngine {
         if working_count > capacity {
             let evict_n = working_count - capacity;
             // Collect the ids of the records to evict (lowest decay = first in sorted list).
-            let evict_ids: Vec<u64> = working_indices[..evict_n]
+            let evict_ids: std::collections::HashSet<u64> = working_indices[..evict_n]
                 .iter()
                 .map(|&i| self.records[i].id)
                 .collect();
@@ -342,7 +341,7 @@ impl ConsolidationEngine {
             });
 
             let evict_n = episodic_count - episodic_capacity;
-            let evict_ids: Vec<u64> = episodic_indices[..evict_n]
+            let evict_ids: std::collections::HashSet<u64> = episodic_indices[..evict_n]
                 .iter()
                 .map(|&i| self.records[i].id)
                 .collect();
@@ -464,7 +463,8 @@ mod tests {
             created_at: 0.0,
             source: MemorySource::User,
         }];
-        let score = ImportanceScorer::score_surprise(&emb, &existing);
+        let existing_refs: Vec<&MemoryRecord> = existing.iter().collect();
+        let score = ImportanceScorer::score_surprise(&emb, &existing_refs);
         assert!(score < 0.01, "expected ~0.0, got {score}");
     }
 

crates/clawhdf5-agent/src/knowledge.rs

@@ -329,47 +329,6 @@ impl KnowledgeCache {
         (id, true)
     }
 
-    // -----------------------------------------------------------------------
-    // Adjacency index (built fresh per traversal call — see doc comment)
-    // -----------------------------------------------------------------------
-
-    /// Build an O(V+R) adjacency index for one traversal call: an entity-id →
-    /// vec-index map for O(1) entity lookups, and an entity-id →
-    /// `(neighbour_id, relation_weight)` map (covering both outgoing and
-    /// incoming edges) for O(1) neighbour expansion. The weight is carried
-    /// alongside each neighbour so callers like `spreading_activation` that
-    /// need per-edge weight don't have to re-scan `relations`.
-    ///
-    /// This is rebuilt at the start of every `bfs_neighbors`/
-    /// `spreading_activation` call rather than cached on the struct: `entities`
-    /// and `relations` are public fields, and `schema.rs`'s deserialization
-    /// path pushes into them directly (bypassing `add_entity`/`add_relation`),
-    /// so a struct-cached index could go stale. Building it once per call
-    /// still turns an O(V·R) (or O(steps·V·R)) traversal into O(V+R) (or
-    /// O(steps·(V+E))), since the old code repeated the O(R) relation scan
-    /// once per visited node instead of once per call.
-    fn build_adjacency(&self) -> (HashMap<u64, usize>, HashMap<u64, Vec<(u64, f32)>>) {
-        let mut entity_index: HashMap<u64, usize> = HashMap::with_capacity(self.entities.len());
-        for (i, e) in self.entities.iter().enumerate() {
-            entity_index.insert(e.id, i);
-        }
-
-        // Note: a self-loop relation (src == tgt) contributes a single
-        // neighbour entry, not two, matching the if/else-if (not two
-        // independent ifs) structure this replaces — otherwise a self-loop
-        // would be double-counted by `spreading_activation`.
-        let mut adjacency: HashMap<u64, Vec<(u64, f32)>> =
-            HashMap::with_capacity(self.relations.len());
-        for r in &self.relations {
-            adjacency.entry(r.src).or_default().push((r.tgt, r.weight));
-            if r.tgt != r.src {
-                adjacency.entry(r.tgt).or_default().push((r.src, r.weight));
-            }
-        }
-
-        (entity_index, adjacency)
-    }
-
     // -----------------------------------------------------------------------
     // Graph traversal: BFS neighbors
     // -----------------------------------------------------------------------
@@ -378,8 +337,6 @@ impl KnowledgeCache {
     /// together with their discovered depth.  The seed entity itself is NOT
     /// included.  Traversal follows both outgoing and incoming relation edges.
     pub fn bfs_neighbors(&self, entity_id: u64, max_depth: usize) -> Vec<(Entity, usize)> {
-        let (entity_index, adjacency) = self.build_adjacency();
-
         let mut visited: HashSet<u64> = HashSet::new();
         let mut queue: VecDeque<(u64, usize)> = VecDeque::new();
         let mut results: Vec<(Entity, usize)> = Vec::new();
@@ -392,15 +349,25 @@ impl KnowledgeCache {
                 continue;
             }
 
-            let Some(neighbours) = adjacency.get(&current_id) else {
-                continue;
-            };
+            // Collect neighbour IDs from outgoing and incoming edges.
+            let neighbours: Vec<u64> = self
+                .relations
+                .iter()
+                .filter_map(|r| {
+                    if r.src == current_id {
+                        Some(r.tgt)
+                    } else if r.tgt == current_id {
+                        Some(r.src)
+                    } else {
+                        None
+                    }
+                })
+                .collect();
 
-            for &(neighbour_id, _weight) in neighbours {
+            for neighbour_id in neighbours {
                 if visited.insert(neighbour_id)
-                    && let Some(&idx) = entity_index.get(&neighbour_id)
+                    && let Some(entity) = self.get_entity(neighbour_id)
                 {
-                    let entity = &self.entities[idx];
                     results.push((entity.clone(), depth + 1));
                     queue.push_back((neighbour_id, depth + 1));
                 }
@@ -472,8 +439,6 @@ impl KnowledgeCache {
         min_activation: f32,
         max_steps: usize,
     ) -> Vec<(u64, f32)> {
-        let (_entity_index, adjacency) = self.build_adjacency();
-
         let mut activation: HashMap<u64, f32> = HashMap::new();
 
         // Initialise seeds with activation 1.0.
@@ -497,11 +462,16 @@ impl KnowledgeCache {
 
             for (source_id, source_score) in current {
                 // Spread to all neighbours via outgoing and incoming edges.
-                let Some(neighbours) = adjacency.get(&source_id) else {
-                    continue;
-                };
-                for &(neighbour_id, weight) in neighbours {
-                    let delta = source_score * weight * decay_factor;
+                for rel in &self.relations {
+                    let neighbour_id = if rel.src == source_id {
+                        rel.tgt
+                    } else if rel.tgt == source_id {
+                        rel.src
+                    } else {
+                        continue;
+                    };
+
+                    let delta = source_score * rel.weight * decay_factor;
                     if delta >= min_activation {
                         *activation.entry(neighbour_id).or_insert(0.0) += delta;
                         any_spread = true;

crates/clawhdf5-agent/src/provenance.rs

@@ -1,31 +1,31 @@
 //! Memory provenance tracking and integrity verification.
 //!
 //! Records the origin, authorship, and a content hash of every memory chunk
-//! so the system can detect content corruption and trace data lineage. The
-//! hash is a SHA-256 digest (see [`hash_content`]), computed via
-//! [`clawhdf5_format::provenance::sha256_hex`]. It is still **unkeyed** — an
-//! actor able to overwrite the stored chunk can also recompute and overwrite
-//! the stored hash alongside it, so this is not an authenticity guarantee
-//! against that threat. What SHA-256 does provide over a fast non-cryptographic
-//! hash (the previous FNV-1a implementation) is collision resistance: an
-//! adversary cannot cheaply craft *different* poisoned content that matches
-//! an already-recorded legitimate hash.
+//! so the system can detect *accidental* corruption and trace data lineage.
+//! The hash is unkeyed (see [`fnv1a_64`]) — this is not a tamper-evidence or
+//! authenticity guarantee.
 
 use std::collections::HashMap;
 
 pub use crate::consolidation::MemorySource;
 
 // ---------------------------------------------------------------------------
-// Hash helper
+// Hash helper (std-only FNV-1a 64-bit)
 // ---------------------------------------------------------------------------
 
-/// SHA-256 hex digest of `text`, used to detect content corruption/tampering.
-///
-/// Unkeyed: an actor able to modify the stored chunk can also recompute and
-/// overwrite the stored hash, so a match is not proof of authenticity — only
-/// that the stored chunk and stored hash are mutually consistent.
-fn hash_content(text: &str) -> String {
-    clawhdf5_format::provenance::sha256_hex(text.as_bytes())
+/// Unkeyed, non-cryptographic FNV-1a hash for detecting accidental content
+/// corruption. It is trivially forgeable by anyone able to modify the stored
+/// data, since they can recompute and overwrite the stored hash alongside
+/// it — do not rely on this as a tamper-evidence or authenticity control.
+fn fnv1a_64(text: &str) -> u64 {
+    const OFFSET: u64 = 14_695_981_039_346_656_037;
+    const PRIME: u64 = 1_099_511_628_211;
+    let mut hash = OFFSET;
+    for byte in text.bytes() {
+        hash ^= byte as u64;
+        hash = hash.wrapping_mul(PRIME);
+    }
+    hash
 }
 
 // ---------------------------------------------------------------------------
@@ -57,8 +57,8 @@ pub struct MemoryProvenance {
     pub created_by: String,
     /// Unix timestamp (seconds) of creation.
     pub created_at: f64,
-    /// SHA-256 hex digest of the chunk text for integrity checking.
-    pub content_hash: String,
+    /// FNV-1a 64-bit hash of the chunk text for integrity checking.
+    pub content_hash: u64,
     pub session_id: String,
     pub verified: bool,
 }
@@ -78,7 +78,7 @@ impl MemoryProvenance {
             source,
             created_by: created_by.into(),
             created_at,
-            content_hash: hash_content(chunk),
+            content_hash: fnv1a_64(chunk),
             session_id: session_id.into(),
             verified: false,
         }
@@ -121,16 +121,13 @@ impl ProvenanceStore {
     /// Re-hash `current_chunk` and compare against the stored hash.
     /// Returns `true` if the content matches (integrity intact).
     ///
-    /// The hash is unkeyed, so an actor able to modify the stored chunk can
-    /// also recompute and overwrite the stored hash. Do not treat a `true`
-    /// result as proof of authenticity against that threat — but unlike a
-    /// non-cryptographic hash, a `false` result reliably indicates that the
-    /// content does not match what was recorded, since SHA-256 makes it
-    /// computationally infeasible to craft different content that collides
-    /// with a specific existing digest.
+    /// This only detects accidental corruption: the hash is unkeyed, so an
+    /// actor able to modify the stored chunk can also recompute and
+    /// overwrite the stored hash. Do not treat a `true` result as proof the
+    /// data hasn't been tampered with.
     pub fn verify_integrity(&self, record_id: u64, current_chunk: &str) -> bool {
         match self.records.get(&record_id) {
-            Some(p) => p.content_hash == hash_content(current_chunk),
+            Some(p) => p.content_hash == fnv1a_64(current_chunk),
             None => false,
         }
     }
@@ -244,32 +241,22 @@ mod tests {
         1_700_000_000.0
     }
 
-    // --- hash_content ---
+    // --- fnv1a_64 ---
 
     #[test]
     fn hash_deterministic() {
-        assert_eq!(hash_content("hello"), hash_content("hello"));
+        assert_eq!(fnv1a_64("hello"), fnv1a_64("hello"));
     }
 
     #[test]
     fn hash_different_inputs() {
-        assert_ne!(hash_content("hello"), hash_content("world"));
+        assert_ne!(fnv1a_64("hello"), fnv1a_64("world"));
     }
 
     #[test]
     fn hash_empty() {
-        // Should not panic, and should match the well-known SHA-256 of the empty string.
-        assert_eq!(
-            hash_content(""),
-            "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
-        );
-    }
-
-    #[test]
-    fn hash_is_sha256_hex() {
-        let h = hash_content("clawhdf5");
-        assert_eq!(h.len(), 64);
-        assert!(h.chars().all(|c| c.is_ascii_hexdigit()));
+        // Should not panic
+        let _ = fnv1a_64("");
     }
 
     // --- MemorySource Display ---
@@ -288,7 +275,7 @@ mod tests {
     #[test]
     fn provenance_new_hashes_chunk() {
         let p = MemoryProvenance::new(1, MemorySource::User, "agent-1", ts(), "hello", "s1");
-        assert_eq!(p.content_hash, hash_content("hello"));
+        assert_eq!(p.content_hash, fnv1a_64("hello"));
         assert!(!p.verified);
     }
 

crates/clawhdf5-format/src/datatype.rs

@@ -204,11 +204,25 @@ fn read_uint(data: &[u8], offset: usize, nbytes: usize) -> Result<u64, FormatErr
     })
 }
 
+/// Maximum recursion depth for nested datatypes (Compound/Enumeration/
+/// VariableLength/Array). A crafted file can nest a message-size-capped
+/// (65535 byte) datatype message ~8000 levels deep, which would blow the
+/// stack — especially on the project's no_std/embedded targets where
+/// available stack is a few KB.
+const MAX_DATATYPE_DEPTH: u16 = 64;
+
 impl Datatype {
     /// Parse a datatype message from raw bytes.
     ///
     /// Returns `(Datatype, bytes_consumed)` for recursive parsing.
     pub fn parse(data: &[u8]) -> Result<(Datatype, usize), FormatError> {
+        Self::parse_with_depth(data, 0)
+    }
+
+    fn parse_with_depth(data: &[u8], depth: u16) -> Result<(Datatype, usize), FormatError> {
+        if depth >= MAX_DATATYPE_DEPTH {
+            return Err(FormatError::NestingDepthExceeded);
+        }
         // Minimum header: 4 bytes (class_and_version + 3 bytes bit field) + 4 bytes size = 8
         ensure_len(data, 0, 8)?;
 
@@ -358,7 +372,7 @@ impl Datatype {
                         pos += name_len;
                         let byte_offset = read_uint(data, pos, ob)?;
                         pos += ob;
-                        let (member_dt, consumed) = Datatype::parse(&data[pos..])?;
+                        let (member_dt, consumed) = Self::parse_with_depth(&data[pos..], depth + 1)?;
                         pos += consumed;
                         members.push(CompoundMember {
                             name,
@@ -384,7 +398,7 @@ impl Datatype {
                         // dimensionality(1) + reserved(3) + dim_perm(4) + 4 dim slots(16) = 24
                         ensure_len(data, pos, 24)?;
                         pos += 24;
-                        let (member_dt, consumed) = Datatype::parse(&data[pos..])?;
+                        let (member_dt, consumed) = Self::parse_with_depth(&data[pos..], depth + 1)?;
                         pos += consumed;
                         members.push(CompoundMember {
                             name,
@@ -415,7 +429,7 @@ impl Datatype {
                 // Enumeration
                 let num_members = (bf0 as u16) | ((bf1 as u16) << 8);
                 // Parse base type
-                let (base_type, base_consumed) = Datatype::parse(&data[pos..])?;
+                let (base_type, base_consumed) = Self::parse_with_depth(&data[pos..], depth + 1)?;
                 pos += base_consumed;
                 let base_size = base_type.type_size();
                 let mut members = Vec::with_capacity(num_members as usize);
@@ -468,7 +482,7 @@ impl Datatype {
                 } else {
                     None
                 };
-                let (base_type, consumed) = Datatype::parse(&data[pos..])?;
+                let (base_type, consumed) = Self::parse_with_depth(&data[pos..], depth + 1)?;
                 pos += consumed;
                 Ok((
                     Datatype::VariableLength {
@@ -494,7 +508,7 @@ impl Datatype {
                     }
                     // skip permutation indices
                     pos += ndims * 4;
-                    let (base_type, consumed) = Datatype::parse(&data[pos..])?;
+                    let (base_type, consumed) = Self::parse_with_depth(&data[pos..], depth + 1)?;
                     pos += consumed;
                     Ok((
                         Datatype::Array {
@@ -515,7 +529,7 @@ impl Datatype {
                         dimensions.push(LittleEndian::read_u32(&data[pos..pos + 4]));
                         pos += 4;
                     }
-                    let (base_type, consumed) = Datatype::parse(&data[pos..])?;
+                    let (base_type, consumed) = Self::parse_with_depth(&data[pos..], depth + 1)?;
                     pos += consumed;
                     Ok((
                         Datatype::Array {
@@ -545,7 +559,7 @@ impl Datatype {
                     pos += name_len;
                     let byte_offset = read_uint(data, pos, ob)?;
                     pos += ob;
-                    let (member_dt, consumed) = Datatype::parse(&data[pos..])?;
+                    let (member_dt, consumed) = Self::parse_with_depth(&data[pos..], depth + 1)?;
                     pos += consumed;
                     members.push(CompoundMember {
                         name,
@@ -814,6 +828,39 @@ mod tests {
         buf
     }
 
+    /// A crafted datatype message nesting Variable-Length wrappers deeper
+    /// than `MAX_DATATYPE_DEPTH` must return `NestingDepthExceeded`
+    /// instead of overflowing the stack.
+    #[test]
+    fn nested_variable_length_exceeds_depth_limit() {
+        // Each VL level is just an 8-byte header (class 9, vl_type=0 =>
+        // sequence, no padding/charset fields) immediately followed by the
+        // next level's bytes, terminated by a fixed-point base type.
+        let levels = MAX_DATATYPE_DEPTH as usize + 10;
+        let mut data = Vec::new();
+        for _ in 0..levels {
+            data.extend_from_slice(&build_dt_header(9, 3, [0, 0, 0], 0));
+        }
+        data.extend_from_slice(&build_fixed_point(4, false, false, 0, 32));
+
+        let result = Datatype::parse(&data);
+        assert!(matches!(result, Err(FormatError::NestingDepthExceeded)));
+    }
+
+    /// A datatype nested just within the depth limit must still parse fine.
+    #[test]
+    fn nested_variable_length_within_depth_limit_ok() {
+        let levels = MAX_DATATYPE_DEPTH as usize - 1;
+        let mut data = Vec::new();
+        for _ in 0..levels {
+            data.extend_from_slice(&build_dt_header(9, 3, [0, 0, 0], 0));
+        }
+        data.extend_from_slice(&build_fixed_point(4, false, false, 0, 32));
+
+        let result = Datatype::parse(&data);
+        assert!(result.is_ok());
+    }
+
     #[test]
     fn test_fixed_point_u8() {
         let data = build_fixed_point(1, false, false, 0, 8);

crates/clawhdf5-format/src/extensible_array.rs

@@ -54,6 +54,19 @@ fn read_offset(data: &[u8], pos: usize, size: u8) -> Result<u64, FormatError> {
     })
 }
 
+fn ensure_len(data: &[u8], offset: usize, needed: usize) -> Result<(), FormatError> {
+    if offset
+        .checked_add(needed)
+        .is_none_or(|end| end > data.len())
+    {
+        return Err(FormatError::UnexpectedEof {
+            expected: offset.saturating_add(needed),
+            available: data.len(),
+        });
+    }
+    Ok(())
+}
+
 fn is_undefined_addr(addr: u64, offset_size: u8) -> bool {
     match offset_size {
         2 => addr == 0xFFFF,
@@ -98,12 +111,7 @@ impl ExtensibleArrayHeader {
         //   6 stats fields (each length_size) + index_block_address(offset_size) + checksum(4)
         let min_size =
             4 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 6 * length_size as usize + offset_size as usize + 4;
-        if offset + min_size > file_data.len() {
-            return Err(FormatError::UnexpectedEof {
-                expected: offset + min_size,
-                available: file_data.len(),
-            });
-        }
+        ensure_len(file_data, offset, min_size)?;
 
         let d = &file_data[offset..];
         if &d[0..4] != b"EAHD" {
@@ -275,12 +283,7 @@ fn read_data_block_elements(
 ) -> Result<Vec<ChunkInfo>, FormatError> {
     // AEDB: signature(4) + version(1) + client_id(1) + header_address(offset_size)
     let db_header_size = 4 + 1 + 1 + offset_size as usize;
-    if db_offset + db_header_size > file_data.len() {
-        return Err(FormatError::UnexpectedEof {
-            expected: db_offset + db_header_size,
-            available: file_data.len(),
-        });
-    }
+    ensure_len(file_data, db_offset, db_header_size)?;
 
     let d = &file_data[db_offset..];
     if &d[0..4] != b"EADB" {
@@ -427,12 +430,7 @@ pub fn read_extensible_array_chunks(
     // Parse index block (AEIB)
     let ib_offset = header.index_block_address as usize;
     let ib_header_size = 4 + 1 + 1 + offset_size as usize; // sig + ver + client + hdr_addr
-    if ib_offset + ib_header_size > file_data.len() {
-        return Err(FormatError::UnexpectedEof {
-            expected: ib_offset + ib_header_size,
-            available: file_data.len(),
-        });
-    }
+    ensure_len(file_data, ib_offset, ib_header_size)?;
 
     let ib = &file_data[ib_offset..];
     if &ib[0..4] != b"EAIB" {
@@ -628,12 +626,7 @@ fn read_super_block(
 
     // AESB: signature(4) + version(1) + client_id(1) + header_address(offset_size)
     let sb_header_size = 4 + 1 + 1 + os;
-    if sb_offset + sb_header_size > file_data.len() {
-        return Err(FormatError::UnexpectedEof {
-            expected: sb_offset + sb_header_size,
-            available: file_data.len(),
-        });
-    }
+    ensure_len(file_data, sb_offset, sb_header_size)?;
 
     if &file_data[sb_offset..sb_offset + 4] != b"EASB" {
         return Err(FormatError::ChunkedReadError(
@@ -759,6 +752,33 @@ mod tests {
         assert!(result.is_err());
     }
 
+    /// A near-`usize::MAX` offset must error cleanly, not overflow/panic.
+    #[test]
+    fn parse_rejects_offset_overflow() {
+        let buf = vec![0u8; 64];
+        let result = ExtensibleArrayHeader::parse(&buf, usize::MAX - 4, 8, 8);
+        assert!(result.is_err());
+    }
+
+    /// A near-`usize::MAX` index block address must error cleanly, not overflow/panic.
+    #[test]
+    fn read_rejects_index_block_offset_overflow() {
+        let header = ExtensibleArrayHeader {
+            client_id: 0,
+            element_size: 8,
+            max_nelmts_bits: 10,
+            idx_blk_elmts: 2,
+            min_dblk_nelmts: 4,
+            super_blk_min_nelmts: 2,
+            max_dblk_nelmts_bits: 8,
+            num_elements: 5,
+            index_block_address: (usize::MAX - 4) as u64,
+        };
+        let buf = vec![0u8; 64];
+        let r = read_extensible_array_chunks(&buf, &header, &[100], &[20], 8, 8, 8);
+        assert!(r.is_err());
+    }
+
     #[test]
     fn parse_header_invalid_version() {
         let mut buf = vec![0u8; 256];

crates/clawhdf5-format/src/fixed_array.rs

@@ -47,6 +47,19 @@ fn read_length(data: &[u8], pos: usize, size: u8) -> Result<u64, FormatError> {
     read_offset(data, pos, size)
 }
 
+fn ensure_len(data: &[u8], offset: usize, needed: usize) -> Result<(), FormatError> {
+    if offset
+        .checked_add(needed)
+        .is_none_or(|end| end > data.len())
+    {
+        return Err(FormatError::UnexpectedEof {
+            expected: offset.saturating_add(needed),
+            available: data.len(),
+        });
+    }
+    Ok(())
+}
+
 fn is_undefined(data: &[u8], pos: usize, size: u8) -> bool {
     let s = size as usize;
     if pos + s > data.len() {
@@ -66,12 +79,7 @@ impl FixedArrayHeader {
         // FAHD signature(4) + version(1) + client_id(1) + element_size(1) +
         // max_nelmts_bits(1) + num_elements(length_size) + data_block_addr(offset_size) + checksum(4)
         let min_size = 4 + 1 + 1 + 1 + 1 + length_size as usize + offset_size as usize + 4;
-        if offset + min_size > file_data.len() {
-            return Err(FormatError::UnexpectedEof {
-                expected: offset + min_size,
-                available: file_data.len(),
-            });
-        }
+        ensure_len(file_data, offset, min_size)?;
 
         let d = &file_data[offset..];
         if &d[0..4] != b"FAHD" {
@@ -126,12 +134,7 @@ pub fn read_fixed_array_chunks(
 
     // Parse data block header: FADB(4) + version(1) + client_id(1) + header_address(offset_size)
     let db_header_size = 4 + 1 + 1 + offset_size as usize;
-    if db_offset + db_header_size > file_data.len() {
-        return Err(FormatError::UnexpectedEof {
-            expected: db_offset + db_header_size,
-            available: file_data.len(),
-        });
-    }
+    ensure_len(file_data, db_offset, db_header_size)?;
 
     let d = &file_data[db_offset..];
     if &d[0..4] != b"FADB" {
@@ -489,6 +492,29 @@ mod tests {
         assert!(r.is_err());
     }
 
+    /// A near-`usize::MAX` offset must error cleanly, not overflow/panic.
+    #[test]
+    fn parse_rejects_offset_overflow() {
+        let buf = vec![0u8; 64];
+        let result = FixedArrayHeader::parse(&buf, usize::MAX - 4, 8, 8);
+        assert!(result.is_err());
+    }
+
+    /// A near-`usize::MAX` data block address must error cleanly, not overflow/panic.
+    #[test]
+    fn read_rejects_data_block_offset_overflow() {
+        let header = FixedArrayHeader {
+            client_id: 0,
+            element_size: 8,
+            max_nelmts_bits: 10,
+            num_elements: 1,
+            data_block_address: (usize::MAX - 4) as u64,
+        };
+        let buf = vec![0u8; 64];
+        let r = read_fixed_array_chunks(&buf, &header, &[100], &[20], 8, 8, 8);
+        assert!(r.is_err());
+    }
+
     #[test]
     fn parse_fixed_array_header_invalid_version() {
         let mut buf = vec![0u8; 256];

crates/clawhdf5-format/src/symbol_table.rs

@@ -80,9 +80,9 @@ impl SymbolTableNode {
         offset_size: u8,
     ) -> Result<SymbolTableNode, FormatError> {
         // signature(4) + version(1) + reserved(1) + number_of_symbols(2) = 8
-        if offset + 8 > file_data.len() {
+        if offset.checked_add(8).is_none_or(|end| end > file_data.len()) {
             return Err(FormatError::UnexpectedEof {
-                expected: offset + 8,
+                expected: offset.saturating_add(8),
                 available: file_data.len(),
             });
         }
@@ -103,7 +103,12 @@ impl SymbolTableNode {
         // Each entry: link_name_offset(os) + obj_hdr_addr(os) + cache_type(4) + reserved(4) + scratch(16)
         let entry_size = os + os + 4 + 4 + 16;
         let entries_start = offset + 8;
-        let needed = entries_start + num_symbols * entry_size;
+        let needed = entries_start
+            .checked_add(num_symbols * entry_size)
+            .ok_or(FormatError::UnexpectedEof {
+                expected: usize::MAX,
+                available: file_data.len(),
+            })?;
         if needed > file_data.len() {
             return Err(FormatError::UnexpectedEof {
                 expected: needed,
@@ -228,4 +233,24 @@ mod tests {
         let err = SymbolTableNode::parse(&data, 0, 8).unwrap_err();
         assert_eq!(err, FormatError::InvalidSymbolTableNodeVersion(2));
     }
+
+    /// A near-`usize::MAX` SNOD offset must error cleanly, not overflow/panic.
+    #[test]
+    fn parse_snod_rejects_offset_overflow() {
+        let data = build_snod(&[], 8);
+        let result = SymbolTableNode::parse(&data, usize::MAX - 4, 8);
+        assert!(result.is_err());
+    }
+
+    /// A huge symbol count combined with a large entries_start must not
+    /// overflow the `needed` size computation.
+    #[test]
+    fn parse_snod_rejects_entries_size_overflow() {
+        let mut data = build_snod(&[], 8);
+        // num_symbols at offset 6..8 — set to max to blow up entries_start + num_symbols*entry_size
+        data[6] = 0xFF;
+        data[7] = 0xFF;
+        let result = SymbolTableNode::parse(&data, usize::MAX / 2, 8);
+        assert!(result.is_err());
+    }
 }

crates/clawhdf5-migrate/src/validate.rs

@@ -144,7 +144,8 @@ fn truncate(s: &str) -> String {
     if s.len() <= 40 {
         s.to_string()
     } else {
-        format!("{}…", &s[..40])
+        let cut = s.char_indices().nth(40).map(|(i, _)| i).unwrap_or(s.len());
+        format!("{}…", &s[..cut])
     }
 }
 
@@ -161,3 +162,31 @@ fn sample_indices(n: usize, full: bool) -> Vec<usize> {
     idx.dedup();
     idx
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn truncate_short_string_unchanged() {
+        assert_eq!(truncate("hello"), "hello");
+    }
+
+    /// A multi-byte character straddling byte offset 40 must not panic a
+    /// byte-index slice — this is arbitrary UTF-8 chunk text from an
+    /// untrusted source database, not test-only input.
+    #[test]
+    fn truncate_multibyte_char_at_boundary_does_not_panic() {
+        // 39 ASCII bytes then a 4-byte emoji straddling the byte-40 cut point.
+        let s = format!("{}{}", "a".repeat(39), "😀".repeat(5));
+        let result = truncate(&s);
+        assert!(result.ends_with('…'));
+        assert!(result.chars().count() < s.chars().count());
+    }
+
+    #[test]
+    fn truncate_exactly_at_limit_unchanged() {
+        let s = "a".repeat(40);
+        assert_eq!(truncate(&s), s);
+    }
+}

research/IMPLEMENTATION_BRIEF.md

@@ -1,176 +1,345 @@
-# ClawHDF5 — Performance / Security / Provenance Implementation Brief
+# Implementation Brief — Performance, Security & Provenance
 
-**Date:** 2026-08-16
-**Scope:** Follow-up hardening pass on top of the already-shipped Tier 1-4 work
-(see `ROADMAP.md` "What's Next" and `IMPROVEMENT_LOG.md`). This brief covers
-only items verified against the current repo state at commit `b2dce41` that
-were **not** already addressed by prior tiers.
+**Phase:** Research
+**Date:** 2026-08-17
+**Scope:** `clawhdf5` Rust workspace (`/mission/repo`)
 
 ## Method
 
-Read `ROADMAP.md`, `IMPROVEMENT_LOG.md`, `IMPROVEMENT_SCAN.md`, and
-`CLAUDE.md` first to avoid re-proposing work already merged (WAL CRC32,
-bounds-check audit + fuzz target, HNSW `prune_connections` parallelism,
-Android JNI length validation, `workspace.dependencies` hoisting, etc. are
-all already done — see those files for the full list).
+Read `ROADMAP.md`, `IMPROVEMENT_LOG.md`, `CLAUDE.md`, `CHANGELOG.md`, and recent
+`git log` before scoping this brief, to avoid re-proposing work already merged.
+The repo has already been through several hardening passes (Tier 1–4, see
+`CHANGELOG.md` "Unreleased" section and the `git log` entries tagged
+`security:`/`perf:`): bounds-check audits on `chunked_read.rs`/`data_read.rs`/
+`local_heap.rs`/`btree_v1.rs`, `MAX_DECOMPRESS_SIZE` output caps, WAL v2
+per-entry CRC32, Android JNI length validation, pyo3 bump, O(1) chunk-cache
+lookup with `Arc`-shared buffers, and optional rayon parallelism for HNSW
+`prune_connections`. None of that is re-proposed here.
 
-Then manually audited:
-- `crates/clawhdf5-format/src/{chunked_read,data_read}.rs` — bounds-check
-  spot audit (sampled `ensure_len` call sites around every raw slice index).
-  **Result: no new gaps found.** Every raw `file_data[a..b]` site sampled is
-  preceded by an `ensure_len`/`read_offset` overflow-checked bound. The prior
-  Tier 4a pass already closed this out.
-- `crates/clawhdf5-agent/src/provenance.rs` — memory record integrity →
-  **gap found**, see INT-01.
-- `crates/clawhdf5-agent/src/knowledge.rs` — knowledge-graph traversal →
-  **gap found**, see INT-02.
-- `crates/clawhdf5-agent/src/bm25.rs` — already has cached IDF, sorted
-  postings, WAND early termination. No changes proposed.
-- `crates/clawhdf5-ann/src/hnsw.rs` — build-time parallelism already scoped
-  to `prune_connections` per Tier 4c; the outer insert loop is flagged in
-  ROADMAP as needing its own correctness-sensitive design pass, out of scope
-  here.
+Four focused audits were run against the areas those passes did **not**
+cover: (1) the HDF5 binary parser files outside the already-audited set, plus
+`clawhdf5-accel`/`clawhdf5-gpu` unsafe code; (2) `clawhdf5-agent`'s
+query-time hot paths (search/rerank/consolidation/knowledge graph); (3) the
+provenance/anomaly-detection subsystem end-to-end; (4) error handling in
+`clawhdf5-io`, `clawhdf5-migrate`, `clawhdf5-py`, and the `clawhdf5` facade.
 
-## INT-01 — Harden agent memory provenance hash from FNV-1a to SHA-256
+`clawhdf5-accel` (SIMD dispatch), `clawhdf5-gpu` (no unsafe code, wgpu-mediated),
+`clawhdf5-io`, `clawhdf5-py`, and the `clawhdf5` facade crate were all found
+already sound for the failure modes investigated — no items proposed for
+those beyond what's listed below. Say so once here rather than padding the
+list with manufactured items.
 
-**File:** `crates/clawhdf5-agent/src/provenance.rs`
-**Category:** Security / Provenance
-**Status:** Implemented this pass.
+---
 
-### Problem
+## Section A — Parser crash safety (crafted-file DoS)
 
-`MemoryProvenance::content_hash` used an unkeyed 64-bit FNV-1a hash
-(`fnv1a_64`) to detect corruption of stored agent-memory chunks. FNV-1a is
-a fast non-cryptographic hash with no collision resistance: an adversary
-attempting to plant poisoned/tampered memory content that still matches a
-previously-recorded or expected hash value only needs to find *any* input
-producing the same 64-bit output, which is computationally cheap for
-FNV-1a (no preimage or collision resistance guarantees at all). Given
-Track 5 of `ROADMAP.md` explicitly claims "poisoning resistance" and
-`verify_integrity()` is the one function whose entire job is to catch
-tampered memory content, using a hash with no collision resistance
-undermines that guarantee in a way that is easy to miss (the doc comment
-already, correctly, disclaims *authenticity* — i.e. it never claimed to
-stop an attacker who can also rewrite the stored hash — but it did not
-protect against a weaker, still-relevant attack: crafting *different*
-poisoned content that collides with an already-recorded legitimate hash).
+These three files use raw `offset + N > file_data.len()` arithmetic instead
+of the `checked_add`-based `ensure_len` helper that every other parser in
+`clawhdf5-format` already uses (established pattern: `btree_v2.rs`,
+`global_heap.rs`, `fractal_heap.rs`, `shared_message.rs`, `local_heap.rs`'s
+own `ensure_len`, etc.). On a crafted file with an address field close to
+`u64::MAX`, the addition overflows — panicking in debug builds, silently
+wrapping in the release profile (no `overflow-checks` set anywhere in the
+workspace `Cargo.toml`), after which the bounds check passes falsely and the
+next slice operation panics anyway. Net effect either way: a crafted file
+crashes the parser instead of returning `Err`.
 
-Separately, `clawhdf5-format` already ships a mature, default-on
-`provenance` feature (`crates/clawhdf5-format/src/provenance.rs`) with a
-`sha256_hex()` helper built on the `sha2` crate, used for on-disk dataset
-provenance attributes. `clawhdf5-agent` already depends on
-`clawhdf5-format` with default features enabled, so `sha256_hex` was
-already reachable with **zero new dependencies**.
+### INT-01 — `crates/clawhdf5-format/src/fixed_array.rs`, `crates/clawhdf5-format/src/extensible_array.rs`
+**Problem:** Six unguarded-addition bounds checks: `FixedArrayHeader::parse`
+(fixed_array.rs:69), the data-block header check in
+`read_fixed_array_chunks` (fixed_array.rs:129), `ExtensibleArrayHeader::parse`
+(extensible_array.rs:101), `read_extensible_array_data_block`
+(extensible_array.rs:278), the index-block parse (extensible_array.rs:429),
+and the super-block parse (extensible_array.rs:630). The offending offsets
+(`data_block_address`/`index_block_address`) come from `DataLayout::parse`
+(`data_layout.rs`, chunk_index_type 3/4 branches, ~lines 460–470), which only
+special-cases the exact all-`0xFF` sentinel via `is_undefined` — any other
+near-max value passes through unchanged.
+**Change:** Replace every raw `offset + N > file_data.len()` in both files
+with the `checked_add`-based `ensure_len` pattern already used elsewhere in
+the crate (e.g. mirror `local_heap.rs`'s `ensure_len`).
 
-### Fix implemented
+### INT-02 — `crates/clawhdf5-format/src/symbol_table.rs`
+**Problem:** `SymbolTableNode::parse` (line 83) uses raw
+`offset + 8 > file_data.len()`, unlike `read_offset` in the same file which
+already uses `checked_add`. `offset` is a SNOD address taken verbatim from a
+v1 B-tree leaf entry and passed straight through by `group_v1.rs:49` with no
+sentinel/range check — a crafted v1-group B-tree leaf with a near-`u64::MAX`
+child pointer overflows the check the same way as INT-01.
+**Change:** Use `offset.checked_add(8)` (`ensure_len` pattern) at line 83.
+Note: the `entries_start + num_symbols * entry_size` addition at line 106 has
+the same raw-arithmetic style, but `num_symbols` is `u16` so the multiply
+itself can't overflow — lower priority, but worth fixing for consistency in
+the same pass.
 
-- `MemoryProvenance::content_hash` changed from `u64` to `String` (lowercase
-  hex SHA-256 digest), computed via `clawhdf5_format::provenance::sha256_hex`.
-- `ProvenanceStore::verify_integrity` now compares SHA-256 hex digests.
-- Removed the local `fnv1a_64` helper from `provenance.rs` (no longer used
-  there — `clawhdf5-agent/src/multimodal.rs` keeps its own independent
-  `fnv1a_64` for `MediaRef` checksums, which is a content-identity/dedup key,
-  not a security/integrity control, so it is intentionally left unchanged
-  and out of scope for this item).
-- Updated the module-level and per-item doc comments to keep the existing,
-  correct disclaimer: this is still an **unkeyed** hash, so it is still not
-  an authenticity/tamper-*evidence* guarantee against an attacker who can
-  rewrite the stored hash alongside the content. What changed is that it is
-  no longer trivially *collidable*, which was the concrete, fixable gap.
-- Updated all existing unit tests in `provenance.rs` for the new `String`
-  hash type; behavior (which records match/mismatch) is unchanged.
+### INT-03 — `crates/clawhdf5-format/src/datatype.rs`
+**Problem:** `Datatype::parse` recurses into itself with no depth counter
+(`grep -n "depth" datatype.rs` — zero hits) for Compound members (lines 361,
+387), Enumeration base type (line 418), VariableLength base type (line 471),
+and Array base type (lines 497, 518). A message data size is capped at
+`u16::MAX` (65535 bytes; see `object_header.rs:141` v1, `object_header.rs:411`
+v2), so a crafted Compound-of-Compound-of-Compound... datatype message can
+nest ~8000 levels deep — enough to blow the stack, and materially worse on
+the project's documented no_std/embedded targets (`thumbv7em-none-eabihf`,
+per `CHANGELOG.md`) where available stack is a few KB. The changelog records
+this exact class of bug already fixed for the N-Bit filter's type tree, but
+that fix was never applied to the general `Datatype::parse` reader used for
+every Dataspace/Attribute/Dataset datatype message.
+**Change:** Thread a `depth: u16` counter through `Datatype::parse`'s
+recursive call sites (mirror `object_header.rs`'s continuation-depth guards)
+and return a new `FormatError::NestingDepthExceeded` past a fixed limit
+(suggest 64).
 
-`MemoryProvenance` and `ProvenanceStore` are only used within
-`clawhdf5-agent` itself (not serialized to the HDF5 format, not consumed by
-other crates), so this is a self-contained, non-breaking-to-other-crates
-change verified by `grep -r MemoryProvenance crates/`.
+---
 
-## INT-02 — Knowledge-graph traversal: replace O(V·R) relation scans with a per-call adjacency index
+## Section B — Provenance & anomaly detection
 
-**File:** `crates/clawhdf5-agent/src/knowledge.rs`
-**Category:** Performance
-**Status:** Implemented this pass.
+The most significant finding of this brief: **the provenance/anomaly
+subsystem exists and is tested, but is never invoked from the real save/load
+path.** It's a fully-built, unused API surface, not an active control.
 
-### Problem
+### INT-04 — `crates/clawhdf5-agent/src/provenance.rs`, `crates/clawhdf5-agent/src/anomaly.rs`, `crates/clawhdf5-agent/src/lib.rs`
+**Problem:** `ProvenanceStore`, `MemoryProvenance::new`, `verify_integrity`,
+`mark_verified`, `WriteAnomalyDetector`, `record_write`,
+`check_pattern_anomaly`, `check_rate_anomaly`, `check_source_anomaly` have
+zero callers outside their own module/tests. `lib.rs` only declares
+`pub mod provenance;` / `pub mod anomaly;` (lines 22, 33) — neither is
+referenced from `HDF5Memory::save_or_update` (~line 495) or the WAL replay
+path (`wal.rs::replay_into_cache`, line 311). Concretely: the 15
+injection-pattern checks, rate limiting, and content-hash integrity
+verification described as shipped in `ROADMAP.md` Track 5 never execute
+during normal library usage today.
+**Change:** Call `ProvenanceStore::add` and
+`WriteAnomalyDetector::record_write` + the `check_*` methods from
+`HDF5Memory::save_or_update`, and call `verify_integrity` from the
+open/load path (surfacing a mismatch to the caller, not panicking). If the
+intent is genuinely opt-in-only, that's a legitimate design choice, but it
+must be documented prominently at the crate root / in `CLAUDE.md` — right
+now it reads as an active control and isn't one.
 
-`KnowledgeCache::bfs_neighbors` and `KnowledgeCache::spreading_activation`
-are the core traversal primitives behind Track 1 (BFS neighbors, subgraph
-extraction) and Track 3 (graph-aware re-ranking) of the agent memory
-system. Both did a **full linear scan over `self.relations`** for every
-node processed:
+### INT-05 — `crates/clawhdf5-agent/src/lib.rs` (`MemoryEntry.source_channel`, ~line 167), `crates/clawhdf5-agent/src/consolidation.rs` (`ConsolidationEngine::add_memory`, ~line 205)
+**Problem:** `source_channel: String` is free text set entirely by the
+caller of `save`/`save_or_update` — nothing validates it against an
+allowlist, so a write can claim `source_channel = "system"` or any other
+privileged-looking label. Separately, `add_memory` takes `source:
+MemorySource` (User/System/Tool/Retrieval/Correction) as a plain parameter;
+`MemorySource::Correction`/`System` get elevated importance weighting in
+`score_correction` (~line 133), so any caller can claim a trust level the
+content doesn't warrant.
+**Change:** Derive `MemorySource`/`source_channel` at the actual trust
+boundary (the ingestion layer that knows the true origin), not as a
+caller-supplied argument to the storage API. At minimum, gate
+`MemorySource::System`/`Correction` construction behind a distinct
+constructor not exposed to the same call path as untrusted content.
 
-- `bfs_neighbors`: for every entity dequeued from the BFS frontier, it
-  scanned the entire `relations: Vec<Relation>` looking for edges touching
-  that entity — O(V·R) instead of O(V+E). It also called
-  `self.get_entity(neighbour_id)`, itself an O(n) linear scan over
-  `entities: Vec<Entity>`, once per newly-discovered neighbour.
-- `spreading_activation`: for every activated node in every propagation
-  step, it likewise scanned all of `self.relations` — O(steps·V·R).
-- `get_subgraph` calls `bfs_neighbors` once per seed, compounding the cost.
+### INT-06 — `crates/clawhdf5-agent/src/anomaly.rs` (`check_pattern_anomaly`, ~lines 192–195)
+**Problem:** Matching is `chunk.to_lowercase().contains(pattern.as_str())` —
+plain literal-substring test after case folding only. Inserting any
+character inside a pattern (extra whitespace, a zero-width character, `.`
+between letters) or substituting a homoglyph for one Latin letter defeats
+every one of the 15 injection patterns; there's no Unicode
+confusable-normalization or punctuation/whitespace stripping.
+**Change:** Normalize input before matching (strip zero-width characters and
+punctuation, apply NFKC + confusable-folding) or switch to fuzzy/token-based
+detection instead of raw `contains`.
 
-For a knowledge graph with thousands of entities/relations (the scale this
-project's own benchmarks target — see `BENCHMARKS.md`), this is
-quadratic-ish behavior in traversal-heavy paths (`get_entity_context`,
-hybrid retrieval re-ranking that pulls graph context) that only gets worse
-as agent memory accumulates over long sessions.
+### INT-07 — `crates/clawhdf5-agent/src/anomaly.rs` (`check_rate_anomaly`, ~lines 149–151)
+**Problem:** The per-minute rate check uses a single global sliding window
+(`self.window.len()`) across all sessions/sources combined. One noisy
+session can trip the shared window without the alert naming the offending
+session (unlike the separate cumulative `max_writes_per_session` check,
+which does name it); conversely, many distinct low-volume sessions can
+jointly flood the shared window without any individual one tripping its own
+per-session limit.
+**Change:** Key the sliding window by session/source (or add a per-source
+rolling count) so the rate check attributes to, and can throttle, the actual
+offender.
 
-### Fix implemented
+### INT-08 — `crates/clawhdf5-format/src/provenance.rs` (`verify_dataset`, ~line 126)
+**Problem:** The SHA-256 content hash is written automatically on save when
+`db.provenance` is set (`file_writer.rs` ~1061–1068, gated on the
+`provenance` feature), but `verify_dataset` is only ever called from test
+files — no reader/open path in `clawhdf5-io` or the `clawhdf5` facade calls
+it. A corrupted dataset is silently readable with no automatic integrity
+check; the write-side machinery exists but nothing consumes it. (Note:
+`CHANGELOG.md` already documents that this hash is unkeyed/tamper-*evident*
+not tamper-*proof* — that's accepted and not re-flagged here; this item is
+about it never being invoked at all, not about its cryptographic strength.)
+**Change:** Optionally call `verify_dataset` on dataset open (behind the
+`provenance` feature) and surface a mismatch as a typed error/warning to the
+caller instead of leaving verification purely opt-in/manual.
 
-Added a private helper, `KnowledgeCache::build_adjacency`, that builds, in
-one O(V+R) pass:
-- `entity_index: HashMap<u64, usize>` — entity id → index into `entities`.
-- `adjacency: HashMap<u64, Vec<u64>>` — entity id → neighbour ids (both
-  outgoing and incoming edges).
+### INT-09 — `crates/clawhdf5-agent/src/wal.rs` (`WalFile::read_entries`, ~lines 219–272)
+**Problem:** Two related gaps. (a) WAL v2's per-entry CRC32 covers only each
+entry's own bytes — there's no sequence number or entry-chaining, so entries
+could be reordered, duplicated, or spliced (e.g. a `Tombstone` moved
+before/after its target `Save`) while every individual entry still passes
+its own CRC check, silently changing replayed cache state. (b) The
+`WAL_VERSION_LEGACY_NO_CRC` branch (~lines 260–266) does no CRC verification
+at all, and the version byte itself is a single unauthenticated byte — since
+`read_entries` is a public standalone API (not just reached via `open()`'s
+one-time migrate-on-read), flipping that byte from `2` to `1` silently
+downgrades every subsequent entry in the file to the fully-unverified
+pre-hardening parser.
+**Change:** Add a monotonic sequence number or entry-chaining (CRC/hash
+including the previous entry's CRC) to detect reordering/splicing. Restrict
+the legacy-no-CRC branch to the `open()` migration path only, or emit a
+warning when `read_entries` falls back to it via any other entry point.
 
-`bfs_neighbors` and `spreading_activation` now build this index **once at
-the top of the call** (not persisted as struct state — see rationale below)
-and use it for O(1) neighbour/entity lookups inside the traversal loop,
-changing the complexity to O(V+E) per call for BFS and O(steps·(V+E)) for
-spreading activation.
+---
 
-**Why not a persistent index on the struct:** `entities`/`relations` are
-public fields, and `crates/clawhdf5-agent/src/schema.rs` (deserialization
-path, loading a persisted knowledge graph back from HDF5) pushes directly
-into `cache.entities`/`cache.relations` rather than going through
-`add_entity`/`add_relation`. A struct-level cached index would silently go
-stale on that path. Building the index fresh at the top of each traversal
-call is O(V+R) — the same asymptotic cost as the scan it replaces would be
-for a *single* node — so it turns what was an O(V·R)-or-worse *whole
-traversal* into an O(V+R) traversal, with no risk of a stale-index
-correctness bug and no change to the existing public API or struct layout.
-`get_entity`, `get_relations_from`, `get_relations_to` are left as-is
-(still O(n)/O(R)): they're public API used elsewhere as one-off lookups,
-not inside a per-node hot loop, so indexing them is lower value and was
-left out of scope to keep this change minimal and low-risk.
+## Section C — Correctness bug (panic on valid, untrusted input)
 
-Existing tests (`test_bfs_neighbors_*`, `test_get_subgraph_*`,
-`test_spreading_activation_*`) exercise correctness and were not modified —
-they pass unchanged, confirming the traversal results are identical to the
-pre-change O(V·R) implementation.
+### INT-10 — `crates/clawhdf5-migrate/src/validate.rs` (`truncate`, lines 143–149)
+**Problem:**
+```rust
+fn truncate(s: &str) -> String {
+    if s.len() <= 40 {
+        s.to_string()
+    } else {
+        format!("{}…", &s[..40])   // byte-index slice, not char-boundary safe
+    }
+}
+```
+`s` is `source.chunk` — arbitrary UTF-8 text read from the source SQLite
+database, called from the chunk-text mismatch branch of `validate_hdf5`
+(~line 58) whenever migrated text doesn't exactly match the source. This is
+the default (non-`--dry-run`) validation path, not test-only code — the file
+has no `#[cfg(test)]` block. If a multi-byte character (emoji, accented
+letter, CJK, etc.) straddles byte offset 40, `&s[..40]` panics with "byte
+index 40 is not a char boundary" instead of producing the diagnostic the
+code exists to report.
+**Change:** Truncate on a char boundary, e.g.
+`let cut = s.char_indices().nth(40).map(|(i, _)| i).unwrap_or(s.len()); format!("{}…", &s[..cut])`.
 
-## Deferred / not implemented this pass
+---
 
-Listed for a future pass — investigated but out of scope for this brief's
-budget, or blocked on a larger design decision already flagged upstream:
+## Section D — Performance (query-time hot paths, `clawhdf5-agent`)
 
-- **HNSW outer insert-loop parallelism** — `ROADMAP.md` already flags this
-  as needing "its own dedicated design pass" before parallelizing; not
-  attempted here to avoid a correctness-sensitive change without that design
-  work.
-- **WAL per-entry format redesign** (explicit length-prefix instead of
-  read-then-verify-CRC32) — `ROADMAP.md` already notes the current CRC32
-  trailer works and this would only be worth revisiting "if profiling shows
-  it matters"; no such profiling signal was found this pass.
-- **`get_entity`/`get_relations_from`/`get_relations_to` indexing** — would
-  further help `get_entity_context` and any other one-off caller, but is
-  lower value than the hot-loop fix in INT-02 and was left out to keep this
-  change minimal.
+`search.rs`, `vector_search.rs`, `hybrid.rs`, `reranker.rs`, `confidence.rs`,
+`temporal.rs`, `ivf.rs`, `pq.rs`, and `gpu_search.rs` were reviewed and found
+already efficient (temporal index uses `partition_point` binary search,
+hybrid merge uses `HashMap` accumulation not nested loops, no gratuitous
+clones in the batch vector paths) — no items proposed there.
 
-## Verification performed
+### INT-11 — `crates/clawhdf5-agent/src/bm25.rs` (`BM25Index::search`, ~lines 118–141)
+**Problem:** The WAND top-k threshold update calls
+`top_k_scores.sort_by(...)` over the full `k`-sized buffer for every matching
+document that beats the running threshold (twice in the `>= k` branch), plus
+another full sort on reaching exactly `k` results. For `m` matching
+documents this is `O(m·k log k)` where a heap gives `O(m log k)`.
+**Change:** Replace `top_k_scores: Vec<f32>` with a min-heap
+(`BinaryHeap<Reverse<f32>>`) of size `k`; pop/push instead of sort-and-index.
 
-- `cargo build --workspace --lib --bins` — clean before starting (baseline).
-- `cargo test --workspace` — run after implementing INT-01 and INT-02 (see
-  commit for pass/fail status).
+### INT-12 — `crates/clawhdf5-agent/src/knowledge.rs` (`KnowledgeCache::resolve_or_create`, lines 304–330)
+**Problem:** `self.entities.iter().map(|e| levenshtein(&lower_name,
+&e.name.to_lowercase()))` allocates a fresh lowercased `String` for every
+entity on every resolution call (this runs per extracted mention during
+entity/relation extraction) and never short-circuits even on an exact
+`dist == 0` match — it scores every remaining entity regardless.
+**Change:** Cache a lowercased name on `Entity` to avoid the
+per-call allocation, and break out of the scan as soon as a `dist == 0`
+match is found.
 
-TASK: INT-01 — Harden agent memory provenance hash from FNV-1a to SHA-256
-TASK: INT-02 — Knowledge-graph traversal: per-call adjacency index instead of O(V·R) relation scans
+### INT-13 — `crates/clawhdf5-agent/src/knowledge.rs` (`bfs_neighbors` lines 339–378, `spreading_activation` lines 435–495, `get_relations_from`/`get_relations_to` lines 247–254)
+**Problem:** All four functions filter/scan the *entire* `self.relations`
+list per node processed (`O(V·E)` for BFS instead of `O(V+E)`;
+`O(max_steps · active_nodes · relations)` for spreading activation), and
+`bfs_neighbors` additionally calls `self.get_entity(neighbour_id)` per
+discovered neighbor, itself an `O(n)` linear `.find()` over `self.entities`.
+**Change:** Build (or maintain incrementally on `add_entity`/`add_relation`)
+a `HashMap<u64, Vec<usize>>` adjacency index and a `HashMap<u64, usize>`
+id→index map, shared across all four functions, replacing the linear scans
+with O(1)/O(degree) lookups.
+
+### INT-14 — `crates/clawhdf5-agent/src/consolidation.rs` (`ConsolidationEngine::add_memory`, lines 212–217)
+**Problem:**
+```rust
+let working: Vec<MemoryRecord> = self.records.iter()
+    .filter(|r| r.tier == MemoryTier::Working)
+    .cloned()
+    .collect();
+```
+`score_surprise` (the only consumer) only reads `r.embedding` by reference —
+the full clone (chunk text + embedding `Vec<f32>`) of every working-tier
+record is discarded immediately after use.
+**Change:** Collect `Vec<&MemoryRecord>` (or iterate the filtered
+`self.records` directly, passing an iterator of `&[f32]`) instead of
+`.cloned()`.
+
+### INT-15 — `crates/clawhdf5-agent/src/consolidation.rs` (`consolidate`, lines 284–291 and 345–351)
+**Problem:** `self.records.retain(|r| !evict_ids.contains(&r.id))` where
+`evict_ids: Vec<u64>` — `retain` calls `.contains()` (linear scan) for every
+record in `self.records`, giving `O(n·m)` cost (n = records, m = eviction
+count) on both the Working-tier eviction (line 289) and Episodic-tier
+eviction (line 350), on every consolidation tick.
+**Change:** Build `evict_ids` as a `HashSet<u64>` for O(1) membership checks.
+
+### INT-16 — `crates/clawhdf5-agent/src/blas_search.rs` (`blas_cosine_batch`, lines 30–39), `crates/clawhdf5-agent/src/accelerate_search.rs` (`accelerate_cosine_batch_vecs`, lines 164–173)
+**Problem:** `cache.embeddings` is stored as `Vec<Vec<f32>>`; both functions
+re-flatten the entire corpus into a fresh `Vec<f32>`
+(`flat.extend_from_slice(&vectors[i])` per non-tombstoned vector) on *every
+single query* before running the actual BLAS/Accelerate matmul — an
+`O(N·dim)` copy paid per query when the `fast-math` feature is enabled. The
+fix pattern already exists in-file: `blas_cosine_batch_flat` (same file,
+lines 89–142) has an `all_active` fast path that skips this copy when
+reading from a pre-flattened buffer directly — it's just not used for the
+`Vec<Vec<f32>>` call sites.
+**Change:** Maintain a persistent flat embedding buffer alongside
+`cache.embeddings` (updated incrementally on insert/delete) and call
+`blas_cosine_batch_flat` instead of `blas_cosine_batch` from both files'
+query paths.
+
+### INT-17 — `crates/clawhdf5-agent/src/entity_extract.rs` (`dedup_overlapping`, lines 302–313)
+**Problem:** `result.iter().any(|existing| ...)` checks every candidate
+entity against all already-accepted entities — `O(n²)` in
+entities-per-extraction-call. This runs at ingestion time (every memory
+save), not query time, and is bounded by entities-per-chunk (typically
+small), so it's lower priority than INT-11 through INT-16.
+**Change:** If profiling shows this matters in practice (large chunks with
+many extracted entities), replace with a spatial/interval-based overlap
+index; otherwise leave as-is — flagging for completeness, not urgency.
+
+---
+
+## Summary table
+
+| INT | Area | File(s) | Category |
+|-----|------|---------|----------|
+| INT-01 | Parser crash safety | `fixed_array.rs`, `extensible_array.rs` | Security |
+| INT-02 | Parser crash safety | `symbol_table.rs` | Security |
+| INT-03 | Parser crash safety | `datatype.rs` | Security |
+| INT-04 | Provenance wiring | `provenance.rs`, `anomaly.rs`, `lib.rs` | Provenance |
+| INT-05 | Source trust boundary | `lib.rs`, `consolidation.rs` | Provenance |
+| INT-06 | Anomaly pattern bypass | `anomaly.rs` | Provenance |
+| INT-07 | Rate-limit attribution | `anomaly.rs` | Provenance |
+| INT-08 | Integrity verification unwired | `clawhdf5-format/provenance.rs` | Provenance |
+| INT-09 | WAL ordering/legacy fallback | `wal.rs` | Provenance |
+| INT-10 | Char-boundary panic | `clawhdf5-migrate/validate.rs` | Correctness |
+| INT-11 | WAND top-k re-sort | `bm25.rs` | Performance |
+| INT-12 | Entity resolution scan | `knowledge.rs` | Performance |
+| INT-13 | Graph traversal scan | `knowledge.rs` | Performance |
+| INT-14 | Unneeded clone | `consolidation.rs` | Performance |
+| INT-15 | O(n·m) eviction | `consolidation.rs` | Performance |
+| INT-16 | Per-query re-flatten | `blas_search.rs`, `accelerate_search.rs` | Performance |
+| INT-17 | O(n²) dedup (low priority) | `entity_extract.rs` | Performance |
+
+## Follow-ups for the coding phase
+
+TASK: INT-01 — Fix unchecked-overflow bounds checks in fixed_array.rs/extensible_array.rs
+TASK: INT-02 — Fix unchecked-overflow bounds check in symbol_table.rs
+TASK: INT-03 — Add recursion-depth guard to Datatype::parse
+TASK: INT-04 — Wire provenance.rs/anomaly.rs into save/load path
+TASK: INT-05 — Enforce source-of-truth for MemorySource/source_channel at trust boundary
+TASK: INT-06 — Harden anomaly pattern matching against whitespace/homoglyph bypass
+TASK: INT-07 — Make anomaly rate-limit window per-source
+TASK: INT-08 — Wire clawhdf5-format provenance verify_dataset into read path
+TASK: INT-09 — Add WAL entry ordering protection and restrict legacy no-CRC fallback
+TASK: INT-10 — Fix byte-index slice panic in clawhdf5-migrate validate.rs truncate()
+TASK: INT-11 — Replace BM25 top-k re-sort with a min-heap
+TASK: INT-12 — Cache lowercased entity names and early-exit in resolve_or_create
+TASK: INT-13 — Add adjacency index for knowledge graph traversal functions
+TASK: INT-14 — Avoid cloning working-tier records in consolidation add_memory
+TASK: INT-15 — Use HashSet for eviction ID membership checks in consolidation
+TASK: INT-16 — Use persistent flat embedding buffer in blas_search/accelerate_search
+TASK: INT-17 — (optional/low-priority) revisit entity_extract dedup_overlapping if profiling shows it matters