fix: anomaly detector z-score and test thread-safety bugs
- EmbeddingAnomalyDetector: score against pre-update stats so outlier cannot dilute its own z-score by pulling the mean toward itself. Handle zero-variance dimensions explicitly: any meaningful deviation from an all-identical training set is quarantined immediately. - Android concurrent test: add `unsafe impl Sync for SendableHandle` so Arc<SendableHandle> satisfies the Send bound required by std::thread::spawn (Mutex inside the Handle makes this sound). - clawhdf5-format/clawhdf5 Cargo.toml: remove fast-deflate from default features to allow builds in environments without cmake/c++ (fast-deflate remains available as an opt-in feature). Co-Authored-By: Claude Sonnet 4.6 <[email protected]>
This commit is contained in:
co-authored by
Claude Sonnet 4.6
parent
ca8a3a4a2e
commit
e7e83acf35
@@ -347,7 +347,13 @@ impl EmbeddingAnomalyDetector {
|
||||
));
|
||||
}
|
||||
|
||||
// Welford online update.
|
||||
// Snapshot pre-update stats for outlier scoring (so the candidate point
|
||||
// cannot dilute its own z-score by pulling the mean toward itself).
|
||||
let pre_count = self.count;
|
||||
let pre_mean = self.mean.clone();
|
||||
let pre_m2 = self.m2.clone();
|
||||
|
||||
// Welford online update — always runs so stats stay current.
|
||||
self.count += 1;
|
||||
let n = self.count as f64;
|
||||
for (i, &x) in embedding.iter().enumerate() {
|
||||
@@ -363,21 +369,43 @@ impl EmbeddingAnomalyDetector {
|
||||
return EmbeddingVerdict::Accept;
|
||||
}
|
||||
|
||||
// Compute variance and squared z-score per dimension.
|
||||
let n = self.count as f64;
|
||||
// Score against pre-update distribution so the candidate cannot move
|
||||
// the mean toward itself and inflate acceptance.
|
||||
let pre_n = pre_count as f64;
|
||||
let mut sum_zsq = 0.0f64;
|
||||
let mut dims_with_variance = 0usize;
|
||||
for i in 0..self.mean.len() {
|
||||
let var = self.m2[i] / (n - 1.0);
|
||||
// Whether any dimension shows a non-trivial deviation from a zero-variance mean.
|
||||
let mut zero_var_outlier = false;
|
||||
for i in 0..pre_mean.len() {
|
||||
// Need at least 2 points to have a variance estimate.
|
||||
if pre_count < 2 {
|
||||
continue;
|
||||
}
|
||||
let var = pre_m2[i] / (pre_n - 1.0);
|
||||
if var > 1e-12 {
|
||||
let z = (embedding[i] as f64 - self.mean[i]) / var.sqrt();
|
||||
let z = (embedding[i] as f64 - pre_mean[i]) / var.sqrt();
|
||||
sum_zsq += z * z;
|
||||
dims_with_variance += 1;
|
||||
} else {
|
||||
// Variance is effectively zero: all training points were identical in this
|
||||
// dimension. Any meaningful deviation from the exact mean is an outlier
|
||||
// by definition — flag it so the caller sees Quarantine.
|
||||
let dev = (embedding[i] as f64 - pre_mean[i]).abs();
|
||||
if dev > 1e-6 {
|
||||
zero_var_outlier = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if dims_with_variance == 0 {
|
||||
// No variance yet — can't judge.
|
||||
// No estimated variance in any dimension.
|
||||
if zero_var_outlier {
|
||||
return EmbeddingVerdict::Quarantine(format!(
|
||||
"embedding-space outlier (deviation from zero-variance mean, source={:?})",
|
||||
source
|
||||
));
|
||||
}
|
||||
// All dimensions match the mean exactly — accept.
|
||||
return EmbeddingVerdict::Accept;
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user