fix: anomaly detector z-score and test thread-safety bugs

- EmbeddingAnomalyDetector: score against pre-update stats so outlier
  cannot dilute its own z-score by pulling the mean toward itself.
  Handle zero-variance dimensions explicitly: any meaningful deviation
  from an all-identical training set is quarantined immediately.
- Android concurrent test: add `unsafe impl Sync for SendableHandle`
  so Arc<SendableHandle> satisfies the Send bound required by
  std::thread::spawn (Mutex inside the Handle makes this sound).
- clawhdf5-format/clawhdf5 Cargo.toml: remove fast-deflate from
  default features to allow builds in environments without cmake/c++
  (fast-deflate remains available as an opt-in feature).

Co-Authored-By: Claude Sonnet 4.6 <[email protected]>
This commit is contained in:
ClawHDF5 Planner
2026-08-12 12:01:37 +00:00
co-authored by Claude Sonnet 4.6
parent ca8a3a4a2e
commit e7e83acf35
4 changed files with 40 additions and 9 deletions
+35 -7
View File
@@ -347,7 +347,13 @@ impl EmbeddingAnomalyDetector {
));
}
// Welford online update.
// Snapshot pre-update stats for outlier scoring (so the candidate point
// cannot dilute its own z-score by pulling the mean toward itself).
let pre_count = self.count;
let pre_mean = self.mean.clone();
let pre_m2 = self.m2.clone();
// Welford online update — always runs so stats stay current.
self.count += 1;
let n = self.count as f64;
for (i, &x) in embedding.iter().enumerate() {
@@ -363,21 +369,43 @@ impl EmbeddingAnomalyDetector {
return EmbeddingVerdict::Accept;
}
// Compute variance and squared z-score per dimension.
let n = self.count as f64;
// Score against pre-update distribution so the candidate cannot move
// the mean toward itself and inflate acceptance.
let pre_n = pre_count as f64;
let mut sum_zsq = 0.0f64;
let mut dims_with_variance = 0usize;
for i in 0..self.mean.len() {
let var = self.m2[i] / (n - 1.0);
// Whether any dimension shows a non-trivial deviation from a zero-variance mean.
let mut zero_var_outlier = false;
for i in 0..pre_mean.len() {
// Need at least 2 points to have a variance estimate.
if pre_count < 2 {
continue;
}
let var = pre_m2[i] / (pre_n - 1.0);
if var > 1e-12 {
let z = (embedding[i] as f64 - self.mean[i]) / var.sqrt();
let z = (embedding[i] as f64 - pre_mean[i]) / var.sqrt();
sum_zsq += z * z;
dims_with_variance += 1;
} else {
// Variance is effectively zero: all training points were identical in this
// dimension. Any meaningful deviation from the exact mean is an outlier
// by definition — flag it so the caller sees Quarantine.
let dev = (embedding[i] as f64 - pre_mean[i]).abs();
if dev > 1e-6 {
zero_var_outlier = true;
}
}
}
if dims_with_variance == 0 {
// No variance yet — can't judge.
// No estimated variance in any dimension.
if zero_var_outlier {
return EmbeddingVerdict::Quarantine(format!(
"embedding-space outlier (deviation from zero-variance mean, source={:?})",
source
));
}
// All dimensions match the mean exactly — accept.
return EmbeddingVerdict::Accept;
}