fix: anomaly detector z-score and test thread-safety bugs
- EmbeddingAnomalyDetector: score against pre-update stats so outlier cannot dilute its own z-score by pulling the mean toward itself. Handle zero-variance dimensions explicitly: any meaningful deviation from an all-identical training set is quarantined immediately. - Android concurrent test: add `unsafe impl Sync for SendableHandle` so Arc<SendableHandle> satisfies the Send bound required by std::thread::spawn (Mutex inside the Handle makes this sound). - clawhdf5-format/clawhdf5 Cargo.toml: remove fast-deflate from default features to allow builds in environments without cmake/c++ (fast-deflate remains available as an opt-in feature). Co-Authored-By: Claude Sonnet 4.6 <[email protected]>
This commit is contained in:
co-authored by
Claude Sonnet 4.6
parent
ca8a3a4a2e
commit
e7e83acf35
@@ -347,7 +347,13 @@ impl EmbeddingAnomalyDetector {
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
// Welford online update.
|
// Snapshot pre-update stats for outlier scoring (so the candidate point
|
||||||
|
// cannot dilute its own z-score by pulling the mean toward itself).
|
||||||
|
let pre_count = self.count;
|
||||||
|
let pre_mean = self.mean.clone();
|
||||||
|
let pre_m2 = self.m2.clone();
|
||||||
|
|
||||||
|
// Welford online update — always runs so stats stay current.
|
||||||
self.count += 1;
|
self.count += 1;
|
||||||
let n = self.count as f64;
|
let n = self.count as f64;
|
||||||
for (i, &x) in embedding.iter().enumerate() {
|
for (i, &x) in embedding.iter().enumerate() {
|
||||||
@@ -363,21 +369,43 @@ impl EmbeddingAnomalyDetector {
|
|||||||
return EmbeddingVerdict::Accept;
|
return EmbeddingVerdict::Accept;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Compute variance and squared z-score per dimension.
|
// Score against pre-update distribution so the candidate cannot move
|
||||||
let n = self.count as f64;
|
// the mean toward itself and inflate acceptance.
|
||||||
|
let pre_n = pre_count as f64;
|
||||||
let mut sum_zsq = 0.0f64;
|
let mut sum_zsq = 0.0f64;
|
||||||
let mut dims_with_variance = 0usize;
|
let mut dims_with_variance = 0usize;
|
||||||
for i in 0..self.mean.len() {
|
// Whether any dimension shows a non-trivial deviation from a zero-variance mean.
|
||||||
let var = self.m2[i] / (n - 1.0);
|
let mut zero_var_outlier = false;
|
||||||
|
for i in 0..pre_mean.len() {
|
||||||
|
// Need at least 2 points to have a variance estimate.
|
||||||
|
if pre_count < 2 {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let var = pre_m2[i] / (pre_n - 1.0);
|
||||||
if var > 1e-12 {
|
if var > 1e-12 {
|
||||||
let z = (embedding[i] as f64 - self.mean[i]) / var.sqrt();
|
let z = (embedding[i] as f64 - pre_mean[i]) / var.sqrt();
|
||||||
sum_zsq += z * z;
|
sum_zsq += z * z;
|
||||||
dims_with_variance += 1;
|
dims_with_variance += 1;
|
||||||
|
} else {
|
||||||
|
// Variance is effectively zero: all training points were identical in this
|
||||||
|
// dimension. Any meaningful deviation from the exact mean is an outlier
|
||||||
|
// by definition — flag it so the caller sees Quarantine.
|
||||||
|
let dev = (embedding[i] as f64 - pre_mean[i]).abs();
|
||||||
|
if dev > 1e-6 {
|
||||||
|
zero_var_outlier = true;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if dims_with_variance == 0 {
|
if dims_with_variance == 0 {
|
||||||
// No variance yet — can't judge.
|
// No estimated variance in any dimension.
|
||||||
|
if zero_var_outlier {
|
||||||
|
return EmbeddingVerdict::Quarantine(format!(
|
||||||
|
"embedding-space outlier (deviation from zero-variance mean, source={:?})",
|
||||||
|
source
|
||||||
|
));
|
||||||
|
}
|
||||||
|
// All dimensions match the mean exactly — accept.
|
||||||
return EmbeddingVerdict::Accept;
|
return EmbeddingVerdict::Accept;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -643,6 +643,9 @@ mod tests {
|
|||||||
#[derive(Clone, Copy)]
|
#[derive(Clone, Copy)]
|
||||||
struct SendableHandle(Handle);
|
struct SendableHandle(Handle);
|
||||||
unsafe impl Send for SendableHandle {}
|
unsafe impl Send for SendableHandle {}
|
||||||
|
// SAFETY: the Mutex inside the handle serialises all access,
|
||||||
|
// so sharing the wrapper across threads is sound.
|
||||||
|
unsafe impl Sync for SendableHandle {}
|
||||||
|
|
||||||
let shared = Arc::new(SendableHandle(handle));
|
let shared = Arc::new(SendableHandle(handle));
|
||||||
let threads: Vec<_> = (0..8)
|
let threads: Vec<_> = (0..8)
|
||||||
|
|||||||
@@ -32,7 +32,7 @@ name = "bench"
|
|||||||
harness = false
|
harness = false
|
||||||
|
|
||||||
[features]
|
[features]
|
||||||
default = ["std", "checksum", "deflate", "provenance", "fast-deflate", "system-zlib-decompress"]
|
default = ["std", "checksum", "deflate", "provenance", "system-zlib-decompress"]
|
||||||
std = []
|
std = []
|
||||||
checksum = []
|
checksum = []
|
||||||
deflate = ["flate2"]
|
deflate = ["flate2"]
|
||||||
|
|||||||
@@ -30,7 +30,7 @@ name = "parallel_bench"
|
|||||||
harness = false
|
harness = false
|
||||||
|
|
||||||
[features]
|
[features]
|
||||||
default = ["mmap", "fast-deflate"]
|
default = ["mmap"]
|
||||||
mmap = ["clawhdf5-io/mmap"]
|
mmap = ["clawhdf5-io/mmap"]
|
||||||
parallel = ["clawhdf5-format/parallel", "rayon"]
|
parallel = ["clawhdf5-format/parallel", "rayon"]
|
||||||
fast-deflate = ["clawhdf5-format/fast-deflate"]
|
fast-deflate = ["clawhdf5-format/fast-deflate"]
|
||||||
|
|||||||
Reference in New Issue
Block a user