perf(agent): replace BM25 WAND top-k re-sort with a min-heap

top_k_scores.sort_by(...) ran over the full k-sized buffer for every
matching document that beat the running threshold (twice in the full
branch), plus another full sort on first reaching k results —
O(m·k log k) for m matching documents. Replace the Vec<f32> buffer
with a BinaryHeap<Reverse<HeapScore>> min-heap of size k, giving
O(m log k). Existing wand_returns_same_results_as_exhaustive test
confirms results are unchanged.

INT-11

crates/clawhdf5-agent/src/bm25.rs

@@ -8,7 +8,28 @@
 //! - Sorted posting lists by doc_id for cache-friendly access
 //! - Block-Max WAND early termination
 
-use std::collections::HashMap;
+use std::cmp::Reverse;
+use std::collections::{BinaryHeap, HashMap};
+
+/// `f32` wrapper providing a total order (via `total_cmp`) so BM25 scores can
+/// be kept in a `BinaryHeap`. Scores are always finite in practice (no NaN
+/// inputs reach this path), so `total_cmp`'s NaN ordering is never exercised.
+#[derive(Debug, Clone, Copy, PartialEq)]
+struct HeapScore(f32);
+
+impl Eq for HeapScore {}
+
+impl PartialOrd for HeapScore {
+    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
+impl Ord for HeapScore {
+    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+        self.0.total_cmp(&other.0)
+    }
+}
 
 /// Default BM25 term-frequency saturation parameter.
 const DEFAULT_K1: f32 = 1.2;
@@ -97,9 +118,11 @@ impl BM25Index {
 
         let total_max_contribution: f32 = max_tf_score.iter().sum();
 
-        // Threshold for WAND early termination
+        // Threshold for WAND early termination. `top_k_heap` is a min-heap of
+        // size k (worst-of-the-top-k at the head) so it can be maintained in
+        // O(log k) per update instead of re-sorting the whole buffer.
         let mut threshold = 0.0f32;
-        let mut top_k_scores: Vec<f32> = Vec::with_capacity(k);
+        let mut top_k_heap: BinaryHeap<Reverse<HeapScore>> = BinaryHeap::with_capacity(k);
 
         for (term_idx, (_, idf, postings)) in query_terms.iter().enumerate() {
             for &(doc_id, freq) in *postings {
@@ -118,24 +141,17 @@ impl BM25Index {
                 if term_idx == query_terms.len() - 1 {
                     // Last term: check if this doc beats threshold
                     let final_score = *entry;
-                    if final_score > threshold && top_k_scores.len() >= k {
+                    if top_k_heap.len() >= k {
-                        // Update threshold
+                        if final_score > threshold {
-                        top_k_scores
+                            // Replace the current worst-of-top-k.
-                            .sort_by(|a, b| b.partial_cmp(a).unwrap_or(std::cmp::Ordering::Equal));
+                            top_k_heap.pop();
-                        if final_score > top_k_scores[k - 1] {
+                            top_k_heap.push(Reverse(HeapScore(final_score)));
-                            top_k_scores[k - 1] = final_score;
+                            threshold = top_k_heap.peek().map(|Reverse(s)| s.0).unwrap_or(0.0);
-                            top_k_scores.sort_by(|a, b| {
-                                b.partial_cmp(a).unwrap_or(std::cmp::Ordering::Equal)
-                            });
-                            threshold = top_k_scores[k - 1];
                         }
-                    } else if top_k_scores.len() < k {
+                    } else {
-                        top_k_scores.push(final_score);
+                        top_k_heap.push(Reverse(HeapScore(final_score)));
-                        if top_k_scores.len() == k {
+                        if top_k_heap.len() == k {
-                            top_k_scores.sort_by(|a, b| {
+                            threshold = top_k_heap.peek().map(|Reverse(s)| s.0).unwrap_or(0.0);
-                                b.partial_cmp(a).unwrap_or(std::cmp::Ordering::Equal)
-                            });
-                            threshold = top_k_scores[k - 1];
                         }
                     }
                 }

crates/clawhdf5-agent/src/consolidation.rs

@@ -118,7 +118,7 @@ impl ImportanceScorer {
 
     /// Novelty score: 1.0 − max cosine similarity against all existing records.
     /// Returns 1.0 when there are no existing memories.
-    pub fn score_surprise(embedding: &[f32], existing_memories: &[MemoryRecord]) -> f32 {
+    pub fn score_surprise(embedding: &[f32], existing_memories: &[&MemoryRecord]) -> f32 {
         if existing_memories.is_empty() {
             return 1.0;
         }
@@ -209,11 +209,10 @@ impl ConsolidationEngine {
         source: MemorySource,
         now: f64,
     ) -> u64 {
-        let working: Vec<MemoryRecord> = self
+        let working: Vec<&MemoryRecord> = self
             .records
             .iter()
             .filter(|r| r.tier == MemoryTier::Working)
-            .cloned()
             .collect();
 
         let surprise = ImportanceScorer::score_surprise(&embedding, &working);
@@ -281,7 +280,7 @@ impl ConsolidationEngine {
         if working_count > capacity {
             let evict_n = working_count - capacity;
             // Collect the ids of the records to evict (lowest decay = first in sorted list).
-            let evict_ids: Vec<u64> = working_indices[..evict_n]
+            let evict_ids: std::collections::HashSet<u64> = working_indices[..evict_n]
                 .iter()
                 .map(|&i| self.records[i].id)
                 .collect();
@@ -342,7 +341,7 @@ impl ConsolidationEngine {
             });
 
             let evict_n = episodic_count - episodic_capacity;
-            let evict_ids: Vec<u64> = episodic_indices[..evict_n]
+            let evict_ids: std::collections::HashSet<u64> = episodic_indices[..evict_n]
                 .iter()
                 .map(|&i| self.records[i].id)
                 .collect();
@@ -464,7 +463,8 @@ mod tests {
             created_at: 0.0,
             source: MemorySource::User,
         }];
-        let score = ImportanceScorer::score_surprise(&emb, &existing);
+        let existing_refs: Vec<&MemoryRecord> = existing.iter().collect();
+        let score = ImportanceScorer::score_surprise(&emb, &existing_refs);
         assert!(score < 0.01, "expected ~0.0, got {score}");
     }
 

crates/clawhdf5-format/src/datatype.rs

@@ -204,11 +204,25 @@ fn read_uint(data: &[u8], offset: usize, nbytes: usize) -> Result<u64, FormatErr
     })
 }
 
+/// Maximum recursion depth for nested datatypes (Compound/Enumeration/
+/// VariableLength/Array). A crafted file can nest a message-size-capped
+/// (65535 byte) datatype message ~8000 levels deep, which would blow the
+/// stack — especially on the project's no_std/embedded targets where
+/// available stack is a few KB.
+const MAX_DATATYPE_DEPTH: u16 = 64;
+
 impl Datatype {
     /// Parse a datatype message from raw bytes.
     ///
     /// Returns `(Datatype, bytes_consumed)` for recursive parsing.
     pub fn parse(data: &[u8]) -> Result<(Datatype, usize), FormatError> {
+        Self::parse_with_depth(data, 0)
+    }
+
+    fn parse_with_depth(data: &[u8], depth: u16) -> Result<(Datatype, usize), FormatError> {
+        if depth >= MAX_DATATYPE_DEPTH {
+            return Err(FormatError::NestingDepthExceeded);
+        }
         // Minimum header: 4 bytes (class_and_version + 3 bytes bit field) + 4 bytes size = 8
         ensure_len(data, 0, 8)?;
 
@@ -358,7 +372,7 @@ impl Datatype {
                         pos += name_len;
                         let byte_offset = read_uint(data, pos, ob)?;
                         pos += ob;
-                        let (member_dt, consumed) = Datatype::parse(&data[pos..])?;
+                        let (member_dt, consumed) = Self::parse_with_depth(&data[pos..], depth + 1)?;
                         pos += consumed;
                         members.push(CompoundMember {
                             name,
@@ -384,7 +398,7 @@ impl Datatype {
                         // dimensionality(1) + reserved(3) + dim_perm(4) + 4 dim slots(16) = 24
                         ensure_len(data, pos, 24)?;
                         pos += 24;
-                        let (member_dt, consumed) = Datatype::parse(&data[pos..])?;
+                        let (member_dt, consumed) = Self::parse_with_depth(&data[pos..], depth + 1)?;
                         pos += consumed;
                         members.push(CompoundMember {
                             name,
@@ -415,7 +429,7 @@ impl Datatype {
                 // Enumeration
                 let num_members = (bf0 as u16) | ((bf1 as u16) << 8);
                 // Parse base type
-                let (base_type, base_consumed) = Datatype::parse(&data[pos..])?;
+                let (base_type, base_consumed) = Self::parse_with_depth(&data[pos..], depth + 1)?;
                 pos += base_consumed;
                 let base_size = base_type.type_size();
                 let mut members = Vec::with_capacity(num_members as usize);
@@ -468,7 +482,7 @@ impl Datatype {
                 } else {
                     None
                 };
-                let (base_type, consumed) = Datatype::parse(&data[pos..])?;
+                let (base_type, consumed) = Self::parse_with_depth(&data[pos..], depth + 1)?;
                 pos += consumed;
                 Ok((
                     Datatype::VariableLength {
@@ -494,7 +508,7 @@ impl Datatype {
                     }
                     // skip permutation indices
                     pos += ndims * 4;
-                    let (base_type, consumed) = Datatype::parse(&data[pos..])?;
+                    let (base_type, consumed) = Self::parse_with_depth(&data[pos..], depth + 1)?;
                     pos += consumed;
                     Ok((
                         Datatype::Array {
@@ -515,7 +529,7 @@ impl Datatype {
                         dimensions.push(LittleEndian::read_u32(&data[pos..pos + 4]));
                         pos += 4;
                     }
-                    let (base_type, consumed) = Datatype::parse(&data[pos..])?;
+                    let (base_type, consumed) = Self::parse_with_depth(&data[pos..], depth + 1)?;
                     pos += consumed;
                     Ok((
                         Datatype::Array {
@@ -545,7 +559,7 @@ impl Datatype {
                     pos += name_len;
                     let byte_offset = read_uint(data, pos, ob)?;
                     pos += ob;
-                    let (member_dt, consumed) = Datatype::parse(&data[pos..])?;
+                    let (member_dt, consumed) = Self::parse_with_depth(&data[pos..], depth + 1)?;
                     pos += consumed;
                     members.push(CompoundMember {
                         name,
@@ -814,6 +828,39 @@ mod tests {
         buf
     }
 
+    /// A crafted datatype message nesting Variable-Length wrappers deeper
+    /// than `MAX_DATATYPE_DEPTH` must return `NestingDepthExceeded`
+    /// instead of overflowing the stack.
+    #[test]
+    fn nested_variable_length_exceeds_depth_limit() {
+        // Each VL level is just an 8-byte header (class 9, vl_type=0 =>
+        // sequence, no padding/charset fields) immediately followed by the
+        // next level's bytes, terminated by a fixed-point base type.
+        let levels = MAX_DATATYPE_DEPTH as usize + 10;
+        let mut data = Vec::new();
+        for _ in 0..levels {
+            data.extend_from_slice(&build_dt_header(9, 3, [0, 0, 0], 0));
+        }
+        data.extend_from_slice(&build_fixed_point(4, false, false, 0, 32));
+
+        let result = Datatype::parse(&data);
+        assert!(matches!(result, Err(FormatError::NestingDepthExceeded)));
+    }
+
+    /// A datatype nested just within the depth limit must still parse fine.
+    #[test]
+    fn nested_variable_length_within_depth_limit_ok() {
+        let levels = MAX_DATATYPE_DEPTH as usize - 1;
+        let mut data = Vec::new();
+        for _ in 0..levels {
+            data.extend_from_slice(&build_dt_header(9, 3, [0, 0, 0], 0));
+        }
+        data.extend_from_slice(&build_fixed_point(4, false, false, 0, 32));
+
+        let result = Datatype::parse(&data);
+        assert!(result.is_ok());
+    }
+
     #[test]
     fn test_fixed_point_u8() {
         let data = build_fixed_point(1, false, false, 0, 8);

crates/clawhdf5-format/src/extensible_array.rs

@@ -54,6 +54,19 @@ fn read_offset(data: &[u8], pos: usize, size: u8) -> Result<u64, FormatError> {
     })
 }
 
+fn ensure_len(data: &[u8], offset: usize, needed: usize) -> Result<(), FormatError> {
+    if offset
+        .checked_add(needed)
+        .is_none_or(|end| end > data.len())
+    {
+        return Err(FormatError::UnexpectedEof {
+            expected: offset.saturating_add(needed),
+            available: data.len(),
+        });
+    }
+    Ok(())
+}
+
 fn is_undefined_addr(addr: u64, offset_size: u8) -> bool {
     match offset_size {
         2 => addr == 0xFFFF,
@@ -98,12 +111,7 @@ impl ExtensibleArrayHeader {
         //   6 stats fields (each length_size) + index_block_address(offset_size) + checksum(4)
         let min_size =
             4 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 6 * length_size as usize + offset_size as usize + 4;
-        if offset + min_size > file_data.len() {
+        ensure_len(file_data, offset, min_size)?;
-            return Err(FormatError::UnexpectedEof {
-                expected: offset + min_size,
-                available: file_data.len(),
-            });
-        }
 
         let d = &file_data[offset..];
         if &d[0..4] != b"EAHD" {
@@ -275,12 +283,7 @@ fn read_data_block_elements(
 ) -> Result<Vec<ChunkInfo>, FormatError> {
     // AEDB: signature(4) + version(1) + client_id(1) + header_address(offset_size)
     let db_header_size = 4 + 1 + 1 + offset_size as usize;
-    if db_offset + db_header_size > file_data.len() {
+    ensure_len(file_data, db_offset, db_header_size)?;
-        return Err(FormatError::UnexpectedEof {
-            expected: db_offset + db_header_size,
-            available: file_data.len(),
-        });
-    }
 
     let d = &file_data[db_offset..];
     if &d[0..4] != b"EADB" {
@@ -427,12 +430,7 @@ pub fn read_extensible_array_chunks(
     // Parse index block (AEIB)
     let ib_offset = header.index_block_address as usize;
     let ib_header_size = 4 + 1 + 1 + offset_size as usize; // sig + ver + client + hdr_addr
-    if ib_offset + ib_header_size > file_data.len() {
+    ensure_len(file_data, ib_offset, ib_header_size)?;
-        return Err(FormatError::UnexpectedEof {
-            expected: ib_offset + ib_header_size,
-            available: file_data.len(),
-        });
-    }
 
     let ib = &file_data[ib_offset..];
     if &ib[0..4] != b"EAIB" {
@@ -628,12 +626,7 @@ fn read_super_block(
 
     // AESB: signature(4) + version(1) + client_id(1) + header_address(offset_size)
     let sb_header_size = 4 + 1 + 1 + os;
-    if sb_offset + sb_header_size > file_data.len() {
+    ensure_len(file_data, sb_offset, sb_header_size)?;
-        return Err(FormatError::UnexpectedEof {
-            expected: sb_offset + sb_header_size,
-            available: file_data.len(),
-        });
-    }
 
     if &file_data[sb_offset..sb_offset + 4] != b"EASB" {
         return Err(FormatError::ChunkedReadError(
@@ -759,6 +752,33 @@ mod tests {
         assert!(result.is_err());
     }
 
+    /// A near-`usize::MAX` offset must error cleanly, not overflow/panic.
+    #[test]
+    fn parse_rejects_offset_overflow() {
+        let buf = vec![0u8; 64];
+        let result = ExtensibleArrayHeader::parse(&buf, usize::MAX - 4, 8, 8);
+        assert!(result.is_err());
+    }
+
+    /// A near-`usize::MAX` index block address must error cleanly, not overflow/panic.
+    #[test]
+    fn read_rejects_index_block_offset_overflow() {
+        let header = ExtensibleArrayHeader {
+            client_id: 0,
+            element_size: 8,
+            max_nelmts_bits: 10,
+            idx_blk_elmts: 2,
+            min_dblk_nelmts: 4,
+            super_blk_min_nelmts: 2,
+            max_dblk_nelmts_bits: 8,
+            num_elements: 5,
+            index_block_address: (usize::MAX - 4) as u64,
+        };
+        let buf = vec![0u8; 64];
+        let r = read_extensible_array_chunks(&buf, &header, &[100], &[20], 8, 8, 8);
+        assert!(r.is_err());
+    }
+
     #[test]
     fn parse_header_invalid_version() {
         let mut buf = vec![0u8; 256];

crates/clawhdf5-format/src/fixed_array.rs

@@ -47,6 +47,19 @@ fn read_length(data: &[u8], pos: usize, size: u8) -> Result<u64, FormatError> {
     read_offset(data, pos, size)
 }
 
+fn ensure_len(data: &[u8], offset: usize, needed: usize) -> Result<(), FormatError> {
+    if offset
+        .checked_add(needed)
+        .is_none_or(|end| end > data.len())
+    {
+        return Err(FormatError::UnexpectedEof {
+            expected: offset.saturating_add(needed),
+            available: data.len(),
+        });
+    }
+    Ok(())
+}
+
 fn is_undefined(data: &[u8], pos: usize, size: u8) -> bool {
     let s = size as usize;
     if pos + s > data.len() {
@@ -66,12 +79,7 @@ impl FixedArrayHeader {
         // FAHD signature(4) + version(1) + client_id(1) + element_size(1) +
         // max_nelmts_bits(1) + num_elements(length_size) + data_block_addr(offset_size) + checksum(4)
         let min_size = 4 + 1 + 1 + 1 + 1 + length_size as usize + offset_size as usize + 4;
-        if offset + min_size > file_data.len() {
+        ensure_len(file_data, offset, min_size)?;
-            return Err(FormatError::UnexpectedEof {
-                expected: offset + min_size,
-                available: file_data.len(),
-            });
-        }
 
         let d = &file_data[offset..];
         if &d[0..4] != b"FAHD" {
@@ -126,12 +134,7 @@ pub fn read_fixed_array_chunks(
 
     // Parse data block header: FADB(4) + version(1) + client_id(1) + header_address(offset_size)
     let db_header_size = 4 + 1 + 1 + offset_size as usize;
-    if db_offset + db_header_size > file_data.len() {
+    ensure_len(file_data, db_offset, db_header_size)?;
-        return Err(FormatError::UnexpectedEof {
-            expected: db_offset + db_header_size,
-            available: file_data.len(),
-        });
-    }
 
     let d = &file_data[db_offset..];
     if &d[0..4] != b"FADB" {
@@ -489,6 +492,29 @@ mod tests {
         assert!(r.is_err());
     }
 
+    /// A near-`usize::MAX` offset must error cleanly, not overflow/panic.
+    #[test]
+    fn parse_rejects_offset_overflow() {
+        let buf = vec![0u8; 64];
+        let result = FixedArrayHeader::parse(&buf, usize::MAX - 4, 8, 8);
+        assert!(result.is_err());
+    }
+
+    /// A near-`usize::MAX` data block address must error cleanly, not overflow/panic.
+    #[test]
+    fn read_rejects_data_block_offset_overflow() {
+        let header = FixedArrayHeader {
+            client_id: 0,
+            element_size: 8,
+            max_nelmts_bits: 10,
+            num_elements: 1,
+            data_block_address: (usize::MAX - 4) as u64,
+        };
+        let buf = vec![0u8; 64];
+        let r = read_fixed_array_chunks(&buf, &header, &[100], &[20], 8, 8, 8);
+        assert!(r.is_err());
+    }
+
     #[test]
     fn parse_fixed_array_header_invalid_version() {
         let mut buf = vec![0u8; 256];

crates/clawhdf5-format/src/symbol_table.rs

@@ -80,9 +80,9 @@ impl SymbolTableNode {
         offset_size: u8,
     ) -> Result<SymbolTableNode, FormatError> {
         // signature(4) + version(1) + reserved(1) + number_of_symbols(2) = 8
-        if offset + 8 > file_data.len() {
+        if offset.checked_add(8).is_none_or(|end| end > file_data.len()) {
             return Err(FormatError::UnexpectedEof {
-                expected: offset + 8,
+                expected: offset.saturating_add(8),
                 available: file_data.len(),
             });
         }
@@ -103,7 +103,12 @@ impl SymbolTableNode {
         // Each entry: link_name_offset(os) + obj_hdr_addr(os) + cache_type(4) + reserved(4) + scratch(16)
         let entry_size = os + os + 4 + 4 + 16;
         let entries_start = offset + 8;
-        let needed = entries_start + num_symbols * entry_size;
+        let needed = entries_start
+            .checked_add(num_symbols * entry_size)
+            .ok_or(FormatError::UnexpectedEof {
+                expected: usize::MAX,
+                available: file_data.len(),
+            })?;
         if needed > file_data.len() {
             return Err(FormatError::UnexpectedEof {
                 expected: needed,
@@ -228,4 +233,24 @@ mod tests {
         let err = SymbolTableNode::parse(&data, 0, 8).unwrap_err();
         assert_eq!(err, FormatError::InvalidSymbolTableNodeVersion(2));
     }
+
+    /// A near-`usize::MAX` SNOD offset must error cleanly, not overflow/panic.
+    #[test]
+    fn parse_snod_rejects_offset_overflow() {
+        let data = build_snod(&[], 8);
+        let result = SymbolTableNode::parse(&data, usize::MAX - 4, 8);
+        assert!(result.is_err());
+    }
+
+    /// A huge symbol count combined with a large entries_start must not
+    /// overflow the `needed` size computation.
+    #[test]
+    fn parse_snod_rejects_entries_size_overflow() {
+        let mut data = build_snod(&[], 8);
+        // num_symbols at offset 6..8 — set to max to blow up entries_start + num_symbols*entry_size
+        data[6] = 0xFF;
+        data[7] = 0xFF;
+        let result = SymbolTableNode::parse(&data, usize::MAX / 2, 8);
+        assert!(result.is_err());
+    }
 }

crates/clawhdf5-migrate/src/validate.rs

@@ -144,7 +144,8 @@ fn truncate(s: &str) -> String {
     if s.len() <= 40 {
         s.to_string()
     } else {
-        format!("{}…", &s[..40])
+        let cut = s.char_indices().nth(40).map(|(i, _)| i).unwrap_or(s.len());
+        format!("{}…", &s[..cut])
     }
 }
 
@@ -161,3 +162,31 @@ fn sample_indices(n: usize, full: bool) -> Vec<usize> {
     idx.dedup();
     idx
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn truncate_short_string_unchanged() {
+        assert_eq!(truncate("hello"), "hello");
+    }
+
+    /// A multi-byte character straddling byte offset 40 must not panic a
+    /// byte-index slice — this is arbitrary UTF-8 chunk text from an
+    /// untrusted source database, not test-only input.
+    #[test]
+    fn truncate_multibyte_char_at_boundary_does_not_panic() {
+        // 39 ASCII bytes then a 4-byte emoji straddling the byte-40 cut point.
+        let s = format!("{}{}", "a".repeat(39), "😀".repeat(5));
+        let result = truncate(&s);
+        assert!(result.ends_with('…'));
+        assert!(result.chars().count() < s.chars().count());
+    }
+
+    #[test]
+    fn truncate_exactly_at_limit_unchanged() {
+        let s = "a".repeat(40);
+        assert_eq!(truncate(&s), s);
+    }
+}

research/IMPLEMENTATION_BRIEF.md

@@ -0,0 +1,345 @@
+# Implementation Brief — Performance, Security & Provenance
+
+**Phase:** Research
+**Date:** 2026-08-17
+**Scope:** `clawhdf5` Rust workspace (`/mission/repo`)
+
+## Method
+
+Read `ROADMAP.md`, `IMPROVEMENT_LOG.md`, `CLAUDE.md`, `CHANGELOG.md`, and recent
+`git log` before scoping this brief, to avoid re-proposing work already merged.
+The repo has already been through several hardening passes (Tier 1–4, see
+`CHANGELOG.md` "Unreleased" section and the `git log` entries tagged
+`security:`/`perf:`): bounds-check audits on `chunked_read.rs`/`data_read.rs`/
+`local_heap.rs`/`btree_v1.rs`, `MAX_DECOMPRESS_SIZE` output caps, WAL v2
+per-entry CRC32, Android JNI length validation, pyo3 bump, O(1) chunk-cache
+lookup with `Arc`-shared buffers, and optional rayon parallelism for HNSW
+`prune_connections`. None of that is re-proposed here.
+
+Four focused audits were run against the areas those passes did **not**
+cover: (1) the HDF5 binary parser files outside the already-audited set, plus
+`clawhdf5-accel`/`clawhdf5-gpu` unsafe code; (2) `clawhdf5-agent`'s
+query-time hot paths (search/rerank/consolidation/knowledge graph); (3) the
+provenance/anomaly-detection subsystem end-to-end; (4) error handling in
+`clawhdf5-io`, `clawhdf5-migrate`, `clawhdf5-py`, and the `clawhdf5` facade.
+
+`clawhdf5-accel` (SIMD dispatch), `clawhdf5-gpu` (no unsafe code, wgpu-mediated),
+`clawhdf5-io`, `clawhdf5-py`, and the `clawhdf5` facade crate were all found
+already sound for the failure modes investigated — no items proposed for
+those beyond what's listed below. Say so once here rather than padding the
+list with manufactured items.
+
+---
+
+## Section A — Parser crash safety (crafted-file DoS)
+
+These three files use raw `offset + N > file_data.len()` arithmetic instead
+of the `checked_add`-based `ensure_len` helper that every other parser in
+`clawhdf5-format` already uses (established pattern: `btree_v2.rs`,
+`global_heap.rs`, `fractal_heap.rs`, `shared_message.rs`, `local_heap.rs`'s
+own `ensure_len`, etc.). On a crafted file with an address field close to
+`u64::MAX`, the addition overflows — panicking in debug builds, silently
+wrapping in the release profile (no `overflow-checks` set anywhere in the
+workspace `Cargo.toml`), after which the bounds check passes falsely and the
+next slice operation panics anyway. Net effect either way: a crafted file
+crashes the parser instead of returning `Err`.
+
+### INT-01 — `crates/clawhdf5-format/src/fixed_array.rs`, `crates/clawhdf5-format/src/extensible_array.rs`
+**Problem:** Six unguarded-addition bounds checks: `FixedArrayHeader::parse`
+(fixed_array.rs:69), the data-block header check in
+`read_fixed_array_chunks` (fixed_array.rs:129), `ExtensibleArrayHeader::parse`
+(extensible_array.rs:101), `read_extensible_array_data_block`
+(extensible_array.rs:278), the index-block parse (extensible_array.rs:429),
+and the super-block parse (extensible_array.rs:630). The offending offsets
+(`data_block_address`/`index_block_address`) come from `DataLayout::parse`
+(`data_layout.rs`, chunk_index_type 3/4 branches, ~lines 460–470), which only
+special-cases the exact all-`0xFF` sentinel via `is_undefined` — any other
+near-max value passes through unchanged.
+**Change:** Replace every raw `offset + N > file_data.len()` in both files
+with the `checked_add`-based `ensure_len` pattern already used elsewhere in
+the crate (e.g. mirror `local_heap.rs`'s `ensure_len`).
+
+### INT-02 — `crates/clawhdf5-format/src/symbol_table.rs`
+**Problem:** `SymbolTableNode::parse` (line 83) uses raw
+`offset + 8 > file_data.len()`, unlike `read_offset` in the same file which
+already uses `checked_add`. `offset` is a SNOD address taken verbatim from a
+v1 B-tree leaf entry and passed straight through by `group_v1.rs:49` with no
+sentinel/range check — a crafted v1-group B-tree leaf with a near-`u64::MAX`
+child pointer overflows the check the same way as INT-01.
+**Change:** Use `offset.checked_add(8)` (`ensure_len` pattern) at line 83.
+Note: the `entries_start + num_symbols * entry_size` addition at line 106 has
+the same raw-arithmetic style, but `num_symbols` is `u16` so the multiply
+itself can't overflow — lower priority, but worth fixing for consistency in
+the same pass.
+
+### INT-03 — `crates/clawhdf5-format/src/datatype.rs`
+**Problem:** `Datatype::parse` recurses into itself with no depth counter
+(`grep -n "depth" datatype.rs` — zero hits) for Compound members (lines 361,
+387), Enumeration base type (line 418), VariableLength base type (line 471),
+and Array base type (lines 497, 518). A message data size is capped at
+`u16::MAX` (65535 bytes; see `object_header.rs:141` v1, `object_header.rs:411`
+v2), so a crafted Compound-of-Compound-of-Compound... datatype message can
+nest ~8000 levels deep — enough to blow the stack, and materially worse on
+the project's documented no_std/embedded targets (`thumbv7em-none-eabihf`,
+per `CHANGELOG.md`) where available stack is a few KB. The changelog records
+this exact class of bug already fixed for the N-Bit filter's type tree, but
+that fix was never applied to the general `Datatype::parse` reader used for
+every Dataspace/Attribute/Dataset datatype message.
+**Change:** Thread a `depth: u16` counter through `Datatype::parse`'s
+recursive call sites (mirror `object_header.rs`'s continuation-depth guards)
+and return a new `FormatError::NestingDepthExceeded` past a fixed limit
+(suggest 64).
+
+---
+
+## Section B — Provenance & anomaly detection
+
+The most significant finding of this brief: **the provenance/anomaly
+subsystem exists and is tested, but is never invoked from the real save/load
+path.** It's a fully-built, unused API surface, not an active control.
+
+### INT-04 — `crates/clawhdf5-agent/src/provenance.rs`, `crates/clawhdf5-agent/src/anomaly.rs`, `crates/clawhdf5-agent/src/lib.rs`
+**Problem:** `ProvenanceStore`, `MemoryProvenance::new`, `verify_integrity`,
+`mark_verified`, `WriteAnomalyDetector`, `record_write`,
+`check_pattern_anomaly`, `check_rate_anomaly`, `check_source_anomaly` have
+zero callers outside their own module/tests. `lib.rs` only declares
+`pub mod provenance;` / `pub mod anomaly;` (lines 22, 33) — neither is
+referenced from `HDF5Memory::save_or_update` (~line 495) or the WAL replay
+path (`wal.rs::replay_into_cache`, line 311). Concretely: the 15
+injection-pattern checks, rate limiting, and content-hash integrity
+verification described as shipped in `ROADMAP.md` Track 5 never execute
+during normal library usage today.
+**Change:** Call `ProvenanceStore::add` and
+`WriteAnomalyDetector::record_write` + the `check_*` methods from
+`HDF5Memory::save_or_update`, and call `verify_integrity` from the
+open/load path (surfacing a mismatch to the caller, not panicking). If the
+intent is genuinely opt-in-only, that's a legitimate design choice, but it
+must be documented prominently at the crate root / in `CLAUDE.md` — right
+now it reads as an active control and isn't one.
+
+### INT-05 — `crates/clawhdf5-agent/src/lib.rs` (`MemoryEntry.source_channel`, ~line 167), `crates/clawhdf5-agent/src/consolidation.rs` (`ConsolidationEngine::add_memory`, ~line 205)
+**Problem:** `source_channel: String` is free text set entirely by the
+caller of `save`/`save_or_update` — nothing validates it against an
+allowlist, so a write can claim `source_channel = "system"` or any other
+privileged-looking label. Separately, `add_memory` takes `source:
+MemorySource` (User/System/Tool/Retrieval/Correction) as a plain parameter;
+`MemorySource::Correction`/`System` get elevated importance weighting in
+`score_correction` (~line 133), so any caller can claim a trust level the
+content doesn't warrant.
+**Change:** Derive `MemorySource`/`source_channel` at the actual trust
+boundary (the ingestion layer that knows the true origin), not as a
+caller-supplied argument to the storage API. At minimum, gate
+`MemorySource::System`/`Correction` construction behind a distinct
+constructor not exposed to the same call path as untrusted content.
+
+### INT-06 — `crates/clawhdf5-agent/src/anomaly.rs` (`check_pattern_anomaly`, ~lines 192–195)
+**Problem:** Matching is `chunk.to_lowercase().contains(pattern.as_str())` —
+plain literal-substring test after case folding only. Inserting any
+character inside a pattern (extra whitespace, a zero-width character, `.`
+between letters) or substituting a homoglyph for one Latin letter defeats
+every one of the 15 injection patterns; there's no Unicode
+confusable-normalization or punctuation/whitespace stripping.
+**Change:** Normalize input before matching (strip zero-width characters and
+punctuation, apply NFKC + confusable-folding) or switch to fuzzy/token-based
+detection instead of raw `contains`.
+
+### INT-07 — `crates/clawhdf5-agent/src/anomaly.rs` (`check_rate_anomaly`, ~lines 149–151)
+**Problem:** The per-minute rate check uses a single global sliding window
+(`self.window.len()`) across all sessions/sources combined. One noisy
+session can trip the shared window without the alert naming the offending
+session (unlike the separate cumulative `max_writes_per_session` check,
+which does name it); conversely, many distinct low-volume sessions can
+jointly flood the shared window without any individual one tripping its own
+per-session limit.
+**Change:** Key the sliding window by session/source (or add a per-source
+rolling count) so the rate check attributes to, and can throttle, the actual
+offender.
+
+### INT-08 — `crates/clawhdf5-format/src/provenance.rs` (`verify_dataset`, ~line 126)
+**Problem:** The SHA-256 content hash is written automatically on save when
+`db.provenance` is set (`file_writer.rs` ~1061–1068, gated on the
+`provenance` feature), but `verify_dataset` is only ever called from test
+files — no reader/open path in `clawhdf5-io` or the `clawhdf5` facade calls
+it. A corrupted dataset is silently readable with no automatic integrity
+check; the write-side machinery exists but nothing consumes it. (Note:
+`CHANGELOG.md` already documents that this hash is unkeyed/tamper-*evident*
+not tamper-*proof* — that's accepted and not re-flagged here; this item is
+about it never being invoked at all, not about its cryptographic strength.)
+**Change:** Optionally call `verify_dataset` on dataset open (behind the
+`provenance` feature) and surface a mismatch as a typed error/warning to the
+caller instead of leaving verification purely opt-in/manual.
+
+### INT-09 — `crates/clawhdf5-agent/src/wal.rs` (`WalFile::read_entries`, ~lines 219–272)
+**Problem:** Two related gaps. (a) WAL v2's per-entry CRC32 covers only each
+entry's own bytes — there's no sequence number or entry-chaining, so entries
+could be reordered, duplicated, or spliced (e.g. a `Tombstone` moved
+before/after its target `Save`) while every individual entry still passes
+its own CRC check, silently changing replayed cache state. (b) The
+`WAL_VERSION_LEGACY_NO_CRC` branch (~lines 260–266) does no CRC verification
+at all, and the version byte itself is a single unauthenticated byte — since
+`read_entries` is a public standalone API (not just reached via `open()`'s
+one-time migrate-on-read), flipping that byte from `2` to `1` silently
+downgrades every subsequent entry in the file to the fully-unverified
+pre-hardening parser.
+**Change:** Add a monotonic sequence number or entry-chaining (CRC/hash
+including the previous entry's CRC) to detect reordering/splicing. Restrict
+the legacy-no-CRC branch to the `open()` migration path only, or emit a
+warning when `read_entries` falls back to it via any other entry point.
+
+---
+
+## Section C — Correctness bug (panic on valid, untrusted input)
+
+### INT-10 — `crates/clawhdf5-migrate/src/validate.rs` (`truncate`, lines 143–149)
+**Problem:**
+```rust
+fn truncate(s: &str) -> String {
+    if s.len() <= 40 {
+        s.to_string()
+    } else {
+        format!("{}…", &s[..40])   // byte-index slice, not char-boundary safe
+    }
+}
+```
+`s` is `source.chunk` — arbitrary UTF-8 text read from the source SQLite
+database, called from the chunk-text mismatch branch of `validate_hdf5`
+(~line 58) whenever migrated text doesn't exactly match the source. This is
+the default (non-`--dry-run`) validation path, not test-only code — the file
+has no `#[cfg(test)]` block. If a multi-byte character (emoji, accented
+letter, CJK, etc.) straddles byte offset 40, `&s[..40]` panics with "byte
+index 40 is not a char boundary" instead of producing the diagnostic the
+code exists to report.
+**Change:** Truncate on a char boundary, e.g.
+`let cut = s.char_indices().nth(40).map(|(i, _)| i).unwrap_or(s.len()); format!("{}…", &s[..cut])`.
+
+---
+
+## Section D — Performance (query-time hot paths, `clawhdf5-agent`)
+
+`search.rs`, `vector_search.rs`, `hybrid.rs`, `reranker.rs`, `confidence.rs`,
+`temporal.rs`, `ivf.rs`, `pq.rs`, and `gpu_search.rs` were reviewed and found
+already efficient (temporal index uses `partition_point` binary search,
+hybrid merge uses `HashMap` accumulation not nested loops, no gratuitous
+clones in the batch vector paths) — no items proposed there.
+
+### INT-11 — `crates/clawhdf5-agent/src/bm25.rs` (`BM25Index::search`, ~lines 118–141)
+**Problem:** The WAND top-k threshold update calls
+`top_k_scores.sort_by(...)` over the full `k`-sized buffer for every matching
+document that beats the running threshold (twice in the `>= k` branch), plus
+another full sort on reaching exactly `k` results. For `m` matching
+documents this is `O(m·k log k)` where a heap gives `O(m log k)`.
+**Change:** Replace `top_k_scores: Vec<f32>` with a min-heap
+(`BinaryHeap<Reverse<f32>>`) of size `k`; pop/push instead of sort-and-index.
+
+### INT-12 — `crates/clawhdf5-agent/src/knowledge.rs` (`KnowledgeCache::resolve_or_create`, lines 304–330)
+**Problem:** `self.entities.iter().map(|e| levenshtein(&lower_name,
+&e.name.to_lowercase()))` allocates a fresh lowercased `String` for every
+entity on every resolution call (this runs per extracted mention during
+entity/relation extraction) and never short-circuits even on an exact
+`dist == 0` match — it scores every remaining entity regardless.
+**Change:** Cache a lowercased name on `Entity` to avoid the
+per-call allocation, and break out of the scan as soon as a `dist == 0`
+match is found.
+
+### INT-13 — `crates/clawhdf5-agent/src/knowledge.rs` (`bfs_neighbors` lines 339–378, `spreading_activation` lines 435–495, `get_relations_from`/`get_relations_to` lines 247–254)
+**Problem:** All four functions filter/scan the *entire* `self.relations`
+list per node processed (`O(V·E)` for BFS instead of `O(V+E)`;
+`O(max_steps · active_nodes · relations)` for spreading activation), and
+`bfs_neighbors` additionally calls `self.get_entity(neighbour_id)` per
+discovered neighbor, itself an `O(n)` linear `.find()` over `self.entities`.
+**Change:** Build (or maintain incrementally on `add_entity`/`add_relation`)
+a `HashMap<u64, Vec<usize>>` adjacency index and a `HashMap<u64, usize>`
+id→index map, shared across all four functions, replacing the linear scans
+with O(1)/O(degree) lookups.
+
+### INT-14 — `crates/clawhdf5-agent/src/consolidation.rs` (`ConsolidationEngine::add_memory`, lines 212–217)
+**Problem:**
+```rust
+let working: Vec<MemoryRecord> = self.records.iter()
+    .filter(|r| r.tier == MemoryTier::Working)
+    .cloned()
+    .collect();
+```
+`score_surprise` (the only consumer) only reads `r.embedding` by reference —
+the full clone (chunk text + embedding `Vec<f32>`) of every working-tier
+record is discarded immediately after use.
+**Change:** Collect `Vec<&MemoryRecord>` (or iterate the filtered
+`self.records` directly, passing an iterator of `&[f32]`) instead of
+`.cloned()`.
+
+### INT-15 — `crates/clawhdf5-agent/src/consolidation.rs` (`consolidate`, lines 284–291 and 345–351)
+**Problem:** `self.records.retain(|r| !evict_ids.contains(&r.id))` where
+`evict_ids: Vec<u64>` — `retain` calls `.contains()` (linear scan) for every
+record in `self.records`, giving `O(n·m)` cost (n = records, m = eviction
+count) on both the Working-tier eviction (line 289) and Episodic-tier
+eviction (line 350), on every consolidation tick.
+**Change:** Build `evict_ids` as a `HashSet<u64>` for O(1) membership checks.
+
+### INT-16 — `crates/clawhdf5-agent/src/blas_search.rs` (`blas_cosine_batch`, lines 30–39), `crates/clawhdf5-agent/src/accelerate_search.rs` (`accelerate_cosine_batch_vecs`, lines 164–173)
+**Problem:** `cache.embeddings` is stored as `Vec<Vec<f32>>`; both functions
+re-flatten the entire corpus into a fresh `Vec<f32>`
+(`flat.extend_from_slice(&vectors[i])` per non-tombstoned vector) on *every
+single query* before running the actual BLAS/Accelerate matmul — an
+`O(N·dim)` copy paid per query when the `fast-math` feature is enabled. The
+fix pattern already exists in-file: `blas_cosine_batch_flat` (same file,
+lines 89–142) has an `all_active` fast path that skips this copy when
+reading from a pre-flattened buffer directly — it's just not used for the
+`Vec<Vec<f32>>` call sites.
+**Change:** Maintain a persistent flat embedding buffer alongside
+`cache.embeddings` (updated incrementally on insert/delete) and call
+`blas_cosine_batch_flat` instead of `blas_cosine_batch` from both files'
+query paths.
+
+### INT-17 — `crates/clawhdf5-agent/src/entity_extract.rs` (`dedup_overlapping`, lines 302–313)
+**Problem:** `result.iter().any(|existing| ...)` checks every candidate
+entity against all already-accepted entities — `O(n²)` in
+entities-per-extraction-call. This runs at ingestion time (every memory
+save), not query time, and is bounded by entities-per-chunk (typically
+small), so it's lower priority than INT-11 through INT-16.
+**Change:** If profiling shows this matters in practice (large chunks with
+many extracted entities), replace with a spatial/interval-based overlap
+index; otherwise leave as-is — flagging for completeness, not urgency.
+
+---
+
+## Summary table
+
+| INT | Area | File(s) | Category |
+|-----|------|---------|----------|
+| INT-01 | Parser crash safety | `fixed_array.rs`, `extensible_array.rs` | Security |
+| INT-02 | Parser crash safety | `symbol_table.rs` | Security |
+| INT-03 | Parser crash safety | `datatype.rs` | Security |
+| INT-04 | Provenance wiring | `provenance.rs`, `anomaly.rs`, `lib.rs` | Provenance |
+| INT-05 | Source trust boundary | `lib.rs`, `consolidation.rs` | Provenance |
+| INT-06 | Anomaly pattern bypass | `anomaly.rs` | Provenance |
+| INT-07 | Rate-limit attribution | `anomaly.rs` | Provenance |
+| INT-08 | Integrity verification unwired | `clawhdf5-format/provenance.rs` | Provenance |
+| INT-09 | WAL ordering/legacy fallback | `wal.rs` | Provenance |
+| INT-10 | Char-boundary panic | `clawhdf5-migrate/validate.rs` | Correctness |
+| INT-11 | WAND top-k re-sort | `bm25.rs` | Performance |
+| INT-12 | Entity resolution scan | `knowledge.rs` | Performance |
+| INT-13 | Graph traversal scan | `knowledge.rs` | Performance |
+| INT-14 | Unneeded clone | `consolidation.rs` | Performance |
+| INT-15 | O(n·m) eviction | `consolidation.rs` | Performance |
+| INT-16 | Per-query re-flatten | `blas_search.rs`, `accelerate_search.rs` | Performance |
+| INT-17 | O(n²) dedup (low priority) | `entity_extract.rs` | Performance |
+
+## Follow-ups for the coding phase
+
+TASK: INT-01 — Fix unchecked-overflow bounds checks in fixed_array.rs/extensible_array.rs
+TASK: INT-02 — Fix unchecked-overflow bounds check in symbol_table.rs
+TASK: INT-03 — Add recursion-depth guard to Datatype::parse
+TASK: INT-04 — Wire provenance.rs/anomaly.rs into save/load path
+TASK: INT-05 — Enforce source-of-truth for MemorySource/source_channel at trust boundary
+TASK: INT-06 — Harden anomaly pattern matching against whitespace/homoglyph bypass
+TASK: INT-07 — Make anomaly rate-limit window per-source
+TASK: INT-08 — Wire clawhdf5-format provenance verify_dataset into read path
+TASK: INT-09 — Add WAL entry ordering protection and restrict legacy no-CRC fallback
+TASK: INT-10 — Fix byte-index slice panic in clawhdf5-migrate validate.rs truncate()
+TASK: INT-11 — Replace BM25 top-k re-sort with a min-heap
+TASK: INT-12 — Cache lowercased entity names and early-exit in resolve_or_create
+TASK: INT-13 — Add adjacency index for knowledge graph traversal functions
+TASK: INT-14 — Avoid cloning working-tier records in consolidation add_memory
+TASK: INT-15 — Use HashSet for eviction ID membership checks in consolidation
+TASK: INT-16 — Use persistent flat embedding buffer in blas_search/accelerate_search
+TASK: INT-17 — (optional/low-priority) revisit entity_extract dedup_overlapping if profiling shows it matters