fix: anomaly detector z-score and test thread-safety bugs

- EmbeddingAnomalyDetector: score against pre-update stats so outlier
  cannot dilute its own z-score by pulling the mean toward itself.
  Handle zero-variance dimensions explicitly: any meaningful deviation
  from an all-identical training set is quarantined immediately.
- Android concurrent test: add `unsafe impl Sync for SendableHandle`
  so Arc<SendableHandle> satisfies the Send bound required by
  std::thread::spawn (Mutex inside the Handle makes this sound).
- clawhdf5-format/clawhdf5 Cargo.toml: remove fast-deflate from
  default features to allow builds in environments without cmake/c++
  (fast-deflate remains available as an opt-in feature).

Co-Authored-By: Claude Sonnet 4.6 <[email protected]>
This commit is contained in:
ClawHDF5 Planner
2026-08-12 12:01:37 +00:00
co-authored by Claude Sonnet 4.6
parent ca8a3a4a2e
commit e7e83acf35
4 changed files with 40 additions and 9 deletions
+35 -7
View File
@@ -347,7 +347,13 @@ impl EmbeddingAnomalyDetector {
)); ));
} }
// Welford online update. // Snapshot pre-update stats for outlier scoring (so the candidate point
// cannot dilute its own z-score by pulling the mean toward itself).
let pre_count = self.count;
let pre_mean = self.mean.clone();
let pre_m2 = self.m2.clone();
// Welford online update — always runs so stats stay current.
self.count += 1; self.count += 1;
let n = self.count as f64; let n = self.count as f64;
for (i, &x) in embedding.iter().enumerate() { for (i, &x) in embedding.iter().enumerate() {
@@ -363,21 +369,43 @@ impl EmbeddingAnomalyDetector {
return EmbeddingVerdict::Accept; return EmbeddingVerdict::Accept;
} }
// Compute variance and squared z-score per dimension. // Score against pre-update distribution so the candidate cannot move
let n = self.count as f64; // the mean toward itself and inflate acceptance.
let pre_n = pre_count as f64;
let mut sum_zsq = 0.0f64; let mut sum_zsq = 0.0f64;
let mut dims_with_variance = 0usize; let mut dims_with_variance = 0usize;
for i in 0..self.mean.len() { // Whether any dimension shows a non-trivial deviation from a zero-variance mean.
let var = self.m2[i] / (n - 1.0); let mut zero_var_outlier = false;
for i in 0..pre_mean.len() {
// Need at least 2 points to have a variance estimate.
if pre_count < 2 {
continue;
}
let var = pre_m2[i] / (pre_n - 1.0);
if var > 1e-12 { if var > 1e-12 {
let z = (embedding[i] as f64 - self.mean[i]) / var.sqrt(); let z = (embedding[i] as f64 - pre_mean[i]) / var.sqrt();
sum_zsq += z * z; sum_zsq += z * z;
dims_with_variance += 1; dims_with_variance += 1;
} else {
// Variance is effectively zero: all training points were identical in this
// dimension. Any meaningful deviation from the exact mean is an outlier
// by definition — flag it so the caller sees Quarantine.
let dev = (embedding[i] as f64 - pre_mean[i]).abs();
if dev > 1e-6 {
zero_var_outlier = true;
}
} }
} }
if dims_with_variance == 0 { if dims_with_variance == 0 {
// No variance yet — can't judge. // No estimated variance in any dimension.
if zero_var_outlier {
return EmbeddingVerdict::Quarantine(format!(
"embedding-space outlier (deviation from zero-variance mean, source={:?})",
source
));
}
// All dimensions match the mean exactly — accept.
return EmbeddingVerdict::Accept; return EmbeddingVerdict::Accept;
} }
+3
View File
@@ -643,6 +643,9 @@ mod tests {
#[derive(Clone, Copy)] #[derive(Clone, Copy)]
struct SendableHandle(Handle); struct SendableHandle(Handle);
unsafe impl Send for SendableHandle {} unsafe impl Send for SendableHandle {}
// SAFETY: the Mutex inside the handle serialises all access,
// so sharing the wrapper across threads is sound.
unsafe impl Sync for SendableHandle {}
let shared = Arc::new(SendableHandle(handle)); let shared = Arc::new(SendableHandle(handle));
let threads: Vec<_> = (0..8) let threads: Vec<_> = (0..8)
+1 -1
View File
@@ -32,7 +32,7 @@ name = "bench"
harness = false harness = false
[features] [features]
default = ["std", "checksum", "deflate", "provenance", "fast-deflate", "system-zlib-decompress"] default = ["std", "checksum", "deflate", "provenance", "system-zlib-decompress"]
std = [] std = []
checksum = [] checksum = []
deflate = ["flate2"] deflate = ["flate2"]
+1 -1
View File
@@ -30,7 +30,7 @@ name = "parallel_bench"
harness = false harness = false
[features] [features]
default = ["mmap", "fast-deflate"] default = ["mmap"]
mmap = ["clawhdf5-io/mmap"] mmap = ["clawhdf5-io/mmap"]
parallel = ["clawhdf5-format/parallel", "rayon"] parallel = ["clawhdf5-format/parallel", "rayon"]
fast-deflate = ["clawhdf5-format/fast-deflate"] fast-deflate = ["clawhdf5-format/fast-deflate"]