Performance, security and provenance hardening (ann/io/migrate/agent) + two audit fixes #2

Merged
osobh merged 23 commits from verify/v3-plus-v6 into main 2026-08-17 14:22:14 +00:00
3 changed files with 171 additions and 25 deletions
Showing only changes of commit 3c7c229e20 - Show all commits
+123 -12
View File
@@ -16,6 +16,55 @@ pub enum MemorySource {
Correction,
}
/// Source classification for content whose true origin is *not*
/// independently verified by the caller of [`ConsolidationEngine::add_memory`]
/// — arbitrary text forwarded from a user, a tool's output, or a retrieval
/// pipeline. This is the only source set `add_memory` accepts; it cannot
/// claim the `System`/`Correction` importance boost (see [`TrustedSource`]
/// and [`ConsolidationEngine::add_trusted_memory`]) — a caller passing
/// through untrusted content has no way to self-report an elevated trust
/// level through this entry point.
#[derive(Clone, Debug, PartialEq)]
pub enum UntrustedSource {
User,
Tool,
Retrieval,
}
impl From<UntrustedSource> for MemorySource {
fn from(s: UntrustedSource) -> Self {
match s {
UntrustedSource::User => MemorySource::User,
UntrustedSource::Tool => MemorySource::Tool,
UntrustedSource::Retrieval => MemorySource::Retrieval,
}
}
}
/// Source classification for content whose elevated trust level has been
/// independently verified by the caller — e.g. the library's own
/// system-generated text, or a caller that ran its own correction-cue
/// detection (as `memory_strategy::SaveOnUserCorrection` does) rather than
/// forwarding a caller-supplied label verbatim. `MemorySource::System`/
/// `Correction` get elevated importance weighting in
/// [`ImportanceScorer::score_correction`]; only reachable through
/// [`ConsolidationEngine::add_trusted_memory`], a distinct entry point from
/// the one untrusted content is passed through.
#[derive(Clone, Debug, PartialEq)]
pub enum TrustedSource {
System,
Correction,
}
impl From<TrustedSource> for MemorySource {
fn from(s: TrustedSource) -> Self {
match s {
TrustedSource::System => MemorySource::System,
TrustedSource::Correction => MemorySource::Correction,
}
}
}
#[derive(Clone, Debug, PartialEq)]
pub enum MemoryTier {
Working,
@@ -199,10 +248,41 @@ impl ConsolidationEngine {
}
}
/// Add a new memory to the Working tier.
/// Add a new memory to the Working tier from an untrusted/ordinary origin
/// (User, Tool, or Retrieval). This is the entry point for arbitrary
/// caller-supplied content — it cannot claim the elevated System/
/// Correction importance boost. Use [`Self::add_trusted_memory`] for
/// content whose elevated trust level the caller has independently
/// verified.
///
/// Importance is scored against existing Working-tier records only.
pub fn add_memory(
&mut self,
chunk: String,
embedding: Vec<f32>,
source: UntrustedSource,
now: f64,
) -> u64 {
self.add_memory_with_source(chunk, embedding, source.into(), now)
}
/// Add a new memory tagged System or Correction, which get elevated
/// importance weighting in [`ImportanceScorer::score_correction`]. Only
/// call this from code that has independently verified the origin (the
/// library's own system-generated text, or a caller that ran its own
/// correction-cue detection) — never from a path that forwards a
/// caller-supplied trust label verbatim.
pub fn add_trusted_memory(
&mut self,
chunk: String,
embedding: Vec<f32>,
source: TrustedSource,
now: f64,
) -> u64 {
self.add_memory_with_source(chunk, embedding, source.into(), now)
}
fn add_memory_with_source(
&mut self,
chunk: String,
embedding: Vec<f32>,
@@ -418,13 +498,44 @@ mod tests {
// ---------------------------------------------------------------------------
// 2. Add memory — basic
// ---------------------------------------------------------------------------
/// add_trusted_memory(TrustedSource::Correction) must actually produce a
/// MemorySource::Correction record — the only way to reach that elevated
/// classification, since add_memory's UntrustedSource has no such variant.
#[test]
fn test_add_trusted_memory_sets_correction_source() {
let mut engine = ConsolidationEngine::new(ConsolidationConfig::default());
let id = engine.add_trusted_memory(
"verified correction".to_string(),
unit_vec(4, 0),
TrustedSource::Correction,
0.0,
);
let rec = engine.get_by_id(id).unwrap();
assert_eq!(rec.source, MemorySource::Correction);
}
/// add_trusted_memory(TrustedSource::System) must produce a
/// MemorySource::System record.
#[test]
fn test_add_trusted_memory_sets_system_source() {
let mut engine = ConsolidationEngine::new(ConsolidationConfig::default());
let id = engine.add_trusted_memory(
"bootstrap text".to_string(),
unit_vec(4, 0),
TrustedSource::System,
0.0,
);
let rec = engine.get_by_id(id).unwrap();
assert_eq!(rec.source, MemorySource::System);
}
#[test]
fn test_add_memory_basic() {
let mut engine = ConsolidationEngine::new(ConsolidationConfig::default());
let id = engine.add_memory(
"Hello world".to_string(),
unit_vec(4, 0),
MemorySource::User,
UntrustedSource::User,
1_000_000.0,
);
assert_eq!(id, 0);
@@ -592,7 +703,7 @@ mod tests {
let id = engine.add_memory(
"x".to_string(),
unit_vec(4, i as usize),
MemorySource::User,
UntrustedSource::User,
i as f64,
);
// Force low importance so promotion threshold is not crossed.
@@ -625,10 +736,10 @@ mod tests {
let cfg = ConsolidationConfig::default();
let mut engine = ConsolidationEngine::new(cfg);
let id = engine.add_memory(
let id = engine.add_trusted_memory(
"important memory".to_string(),
unit_vec(4, 0),
MemorySource::Correction,
TrustedSource::Correction,
0.0,
);
// Force importance above threshold.
@@ -661,7 +772,7 @@ mod tests {
let id = engine.add_memory(
"frequently accessed".to_string(),
unit_vec(4, 0),
MemorySource::User,
UntrustedSource::User,
0.0,
);
@@ -689,7 +800,7 @@ mod tests {
#[test]
fn test_access_memory_reactivation() {
let mut engine = ConsolidationEngine::new(ConsolidationConfig::default());
let id = engine.add_memory("chunk".to_string(), unit_vec(4, 0), MemorySource::User, 0.0);
let id = engine.add_memory("chunk".to_string(), unit_vec(4, 0), UntrustedSource::User, 0.0);
engine.access_memory(id, 5000.0);
let rec = engine.get_by_id(id).unwrap();
@@ -710,11 +821,11 @@ mod tests {
let mut engine = ConsolidationEngine::new(ConsolidationConfig::default());
// 2 Working
engine.add_memory("w1".to_string(), unit_vec(4, 0), MemorySource::User, 0.0);
engine.add_memory("w2".to_string(), unit_vec(4, 1), MemorySource::User, 0.0);
engine.add_memory("w1".to_string(), unit_vec(4, 0), UntrustedSource::User, 0.0);
engine.add_memory("w2".to_string(), unit_vec(4, 1), UntrustedSource::User, 0.0);
// 1 Episodic (manually set)
let id_e = engine.add_memory("e1".to_string(), unit_vec(4, 2), MemorySource::User, 0.0);
let id_e = engine.add_memory("e1".to_string(), unit_vec(4, 2), UntrustedSource::User, 0.0);
engine
.records
.iter_mut()
@@ -723,7 +834,7 @@ mod tests {
.tier = MemoryTier::Episodic;
// 1 Semantic (manually set)
let id_s = engine.add_memory("s1".to_string(), unit_vec(4, 3), MemorySource::User, 0.0);
let id_s = engine.add_memory("s1".to_string(), unit_vec(4, 3), UntrustedSource::User, 0.0);
engine
.records
.iter_mut()
@@ -752,7 +863,7 @@ mod tests {
let id = engine.add_memory(
"episodic chunk".to_string(),
unit_vec(4, i as usize),
MemorySource::User,
UntrustedSource::User,
i as f64,
);
let rec = engine.records.iter_mut().find(|r| r.id == id).unwrap();
+39 -6
View File
@@ -349,18 +349,24 @@ impl HDF5Memory {
// ---- Provenance & anomaly detection ------------------------------------
//
// Heuristic, best-effort session bookkeeping: a coarse MemorySource
// inferred from the caller-supplied source_channel string (not a trust
// boundary — see consolidation::MemorySource for the gated construction
// path), a content hash per record for detecting accidental in-session
// corruption, and write-pattern anomaly checks (rate, injection-pattern,
// inferred from the caller-supplied source_channel string, a content
// hash per record for detecting accidental in-session corruption, and
// write-pattern anomaly checks (rate, injection-pattern,
// source-distribution skew) run on every save/update.
/// Infer a coarse `MemorySource` from a free-text `source_channel` for
/// provenance/anomaly bookkeeping purposes only.
///
/// `source_channel` is caller-supplied and unvalidated (`MemoryEntry` has
/// no trust field), so this deliberately never returns `System` or
/// `Correction` — those are consolidation::MemorySource's elevated
/// classifications (see `UntrustedSource`/`TrustedSource`), and inferring
/// them from a string the caller controls would let a write dodge
/// `check_source_anomaly`'s User-flood detection by simply labeling
/// itself `source_channel = "system"`. Everything not recognized as
/// `Tool`/`Retrieval` is conservatively bucketed as `User`.
fn infer_memory_source(source_channel: &str) -> consolidation::MemorySource {
match source_channel {
"correction" => consolidation::MemorySource::Correction,
"system" => consolidation::MemorySource::System,
"tool" => consolidation::MemorySource::Tool,
"retrieval" => consolidation::MemorySource::Retrieval,
_ => consolidation::MemorySource::User,
@@ -909,6 +915,33 @@ mod tests {
assert!(!mem.provenance.verify_integrity(idx as u64, "tampered"));
}
/// A caller cannot dodge check_source_anomaly's User-flood detection by
/// self-labeling source_channel = "system" — infer_memory_source must
/// never grant the elevated System/Correction classification from
/// unvalidated caller-supplied text.
#[test]
fn source_channel_cannot_claim_system_to_evade_source_anomaly() {
let dir = TempDir::new().unwrap();
let config = make_config(&dir);
let mut mem = HDF5Memory::create(config).unwrap();
for i in 0..15 {
let mut entry = make_entry(&format!("flood {i}"), &[1.0, 0.0, 0.0, 0.0]);
entry.source_channel = "system".to_owned();
entry.timestamp = 1000000.0 + i as f64;
mem.save(entry).unwrap();
}
let alerts = mem.take_anomaly_alerts();
assert!(
alerts
.iter()
.any(|a| a.message.contains("source distribution")),
"a flood of writes claiming source_channel=\"system\" must still trigger \
source-distribution anomaly detection as User-sourced, got: {alerts:?}"
);
}
/// A chunk containing a known injection pattern must raise a queued
/// anomaly alert through the real save path, not just in anomaly.rs's
/// own unit tests.
@@ -22,7 +22,9 @@
use std::time::Instant;
use clawhdf5_agent::bm25::BM25Index;
use clawhdf5_agent::consolidation::{ConsolidationConfig, ConsolidationEngine, MemorySource};
use clawhdf5_agent::consolidation::{
ConsolidationConfig, ConsolidationEngine, TrustedSource, UntrustedSource,
};
use clawhdf5_agent::hybrid::hybrid_search;
const EMBEDDING_DIM: usize = 384;
@@ -232,7 +234,7 @@ fn run_quality_benchmark() {
for i in 0..SIGNAL_KEYWORDS.len() {
let chunk = make_signal_content(i);
let embedding = make_embedding(i * 1000);
let id = engine.add_memory(chunk, embedding, MemorySource::Correction, now);
let id = engine.add_trusted_memory(chunk, embedding, TrustedSource::Correction, now);
signal_ids.push(id);
}
@@ -240,7 +242,7 @@ fn run_quality_benchmark() {
for i in 0..990 {
let chunk = make_noise_content(i);
let embedding = make_embedding(i + 100);
engine.add_memory(chunk, embedding, MemorySource::System, now + i as f64 * 0.1);
engine.add_trusted_memory(chunk, embedding, TrustedSource::System, now + i as f64 * 0.1);
}
println!(" → Inserted {} records total", engine.records().len());
@@ -333,7 +335,7 @@ fn run_cycle_time_benchmark() {
for i in 0..n {
let chunk = make_noise_content(i);
let embedding = make_embedding(i);
engine.add_memory(chunk, embedding, MemorySource::User, now + i as f64);
engine.add_memory(chunk, embedding, UntrustedSource::User, now + i as f64);
}
// Warmup
@@ -344,7 +346,7 @@ fn run_cycle_time_benchmark() {
for i in n..(n * 2) {
let chunk = make_noise_content(i);
let embedding = make_embedding(i);
engine.add_memory(chunk, embedding, MemorySource::User, now + i as f64);
engine.add_memory(chunk, embedding, UntrustedSource::User, now + i as f64);
}
// Timed consolidation
@@ -410,13 +412,13 @@ fn run_memory_reduction_benchmark() {
for i in 0..signal_count {
let chunk = make_signal_content(i % SIGNAL_KEYWORDS.len());
let emb = make_embedding(i * 999);
let id = engine.add_memory(chunk, emb, MemorySource::Correction, now);
let id = engine.add_trusted_memory(chunk, emb, TrustedSource::Correction, now);
signal_ids.push(id);
}
for i in 0..noise_count {
let chunk = make_noise_content(i);
let emb = make_embedding(i + 200);
engine.add_memory(chunk, emb, MemorySource::System, now + i as f64 * 0.1);
engine.add_trusted_memory(chunk, emb, TrustedSource::System, now + i as f64 * 0.1);
}
// Access signal records heavily