1189 lines
36 KiB
Rust
1189 lines
36 KiB
Rust
//! Anomaly detection algorithms including z-score, IQR, and isolation forest
|
|
//!
|
|
//! This module provides comprehensive anomaly detection capabilities using various
|
|
//! statistical and machine learning methods for both univariate and multivariate
|
|
//! anomaly detection.
|
|
|
|
use std::collections::HashMap;
|
|
|
|
use rand::prelude::*;
|
|
use serde::{Deserialize, Serialize};
|
|
|
|
use crate::{DataValue, Result, ValidationError};
|
|
|
|
/// Extension trait for statistical methods on Vec<f64>
|
|
trait StatisticalExtensions {
|
|
fn median(&self) -> f64;
|
|
fn quantile(&self, q: f64) -> f64;
|
|
fn mean(&self) -> f64;
|
|
fn variance(&self) -> f64;
|
|
}
|
|
|
|
impl StatisticalExtensions for [f64] {
|
|
fn median(&self) -> f64 {
|
|
if self.is_empty() {
|
|
return 0.0;
|
|
}
|
|
|
|
let mut sorted = self.to_vec();
|
|
sorted.sort_by(|a, b| a.total_cmp(b));
|
|
|
|
let len = sorted.len();
|
|
if len % 2 == 0 {
|
|
(sorted[len / 2 - 1] + sorted[len / 2]) / 2.0
|
|
} else {
|
|
sorted[len / 2]
|
|
}
|
|
}
|
|
|
|
fn quantile(&self, q: f64) -> f64 {
|
|
if self.is_empty() {
|
|
return 0.0;
|
|
}
|
|
|
|
let mut sorted = self.to_vec();
|
|
sorted.sort_by(|a, b| a.total_cmp(b));
|
|
|
|
let index = (q * (sorted.len() - 1) as f64).round() as usize;
|
|
sorted[index.min(sorted.len() - 1)]
|
|
}
|
|
|
|
fn mean(&self) -> f64 {
|
|
if self.is_empty() {
|
|
return 0.0;
|
|
}
|
|
self.iter().sum::<f64>() / self.len() as f64
|
|
}
|
|
|
|
fn variance(&self) -> f64 {
|
|
if self.len() < 2 {
|
|
return 0.0;
|
|
}
|
|
|
|
let mean_val = StatisticalExtensions::mean(self);
|
|
let sum_sq_diff: f64 = self.iter().map(|x| (x - mean_val).powi(2)).sum();
|
|
sum_sq_diff / (self.len() - 1) as f64
|
|
}
|
|
}
|
|
|
|
/// Main anomaly detector with multiple detection methods
|
|
#[derive(Debug, Clone)]
|
|
pub struct AnomalyDetector {
|
|
/// Configuration for anomaly detection
|
|
pub config: AnomalyConfig,
|
|
/// Z-score detector
|
|
pub z_score_detector: ZScoreDetector,
|
|
/// IQR-based detector
|
|
pub iqr_detector: IQRDetector,
|
|
/// Isolation Forest detector
|
|
pub isolation_forest_detector: IsolationForestDetector,
|
|
/// Historical data for comparison
|
|
pub historical_data: HashMap<String, Vec<f64>>,
|
|
}
|
|
|
|
/// Configuration for anomaly detection
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct AnomalyConfig {
|
|
/// Z-score threshold (default: 3.0)
|
|
pub z_score_threshold: f64,
|
|
/// IQR multiplier (default: 1.5)
|
|
pub iqr_multiplier: f64,
|
|
/// Isolation forest contamination rate (default: 0.1)
|
|
pub contamination_rate: f64,
|
|
/// Number of trees in isolation forest (default: 100)
|
|
pub n_trees: usize,
|
|
/// Subsample size for isolation forest (default: 256)
|
|
pub subsample_size: usize,
|
|
/// Enable multivariate detection
|
|
pub enable_multivariate: bool,
|
|
/// Minimum samples required for detection
|
|
pub min_samples: usize,
|
|
/// Enable streaming detection
|
|
pub enable_streaming: bool,
|
|
/// Window size for streaming detection
|
|
pub streaming_window_size: usize,
|
|
}
|
|
|
|
/// Result of anomaly detection
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct AnomalyResult {
|
|
/// Field name where anomaly was detected
|
|
pub field_name: String,
|
|
/// Value that was flagged as anomalous
|
|
pub anomalous_value: String,
|
|
/// Anomaly score (higher = more anomalous)
|
|
pub anomaly_score: f64,
|
|
/// Detection method used
|
|
pub detection_method: DetectionMethod,
|
|
/// Threshold used for detection
|
|
pub threshold: f64,
|
|
/// Statistical context
|
|
pub context: AnomalyContext,
|
|
/// Severity of the anomaly
|
|
pub severity: AnomalySeverity,
|
|
/// Confidence level of the detection
|
|
pub confidence: f64,
|
|
}
|
|
|
|
/// Different anomaly detection methods
|
|
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
|
pub enum DetectionMethod {
|
|
/// Z-score based detection
|
|
ZScore,
|
|
/// Interquartile range based detection
|
|
IQR,
|
|
/// Isolation forest detection
|
|
IsolationForest,
|
|
/// Modified Z-score (using median absolute deviation)
|
|
ModifiedZScore,
|
|
/// Local Outlier Factor
|
|
LocalOutlierFactor,
|
|
/// Statistical process control
|
|
StatisticalProcessControl,
|
|
/// Seasonal decomposition
|
|
SeasonalDecomposition,
|
|
/// Ensemble method (multiple detectors)
|
|
Ensemble,
|
|
}
|
|
|
|
/// Context information for anomaly detection
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct AnomalyContext {
|
|
/// Statistical mean of the data
|
|
pub mean: f64,
|
|
/// Statistical standard deviation
|
|
pub std_dev: f64,
|
|
/// Median value
|
|
pub median: f64,
|
|
/// First quartile (25th percentile)
|
|
pub q1: f64,
|
|
/// Third quartile (75th percentile)
|
|
pub q3: f64,
|
|
/// Interquartile range
|
|
pub iqr: f64,
|
|
/// Minimum value in dataset
|
|
pub min: f64,
|
|
/// Maximum value in dataset
|
|
pub max: f64,
|
|
/// Number of samples used
|
|
pub sample_size: usize,
|
|
}
|
|
|
|
/// Severity levels for anomalies
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
|
|
pub enum AnomalySeverity {
|
|
/// Low severity anomaly
|
|
Low,
|
|
/// Medium severity anomaly
|
|
Medium,
|
|
/// High severity anomaly
|
|
High,
|
|
/// Critical severity anomaly
|
|
Critical,
|
|
}
|
|
|
|
/// Z-score based anomaly detector
|
|
#[derive(Debug, Clone)]
|
|
pub struct ZScoreDetector {
|
|
/// Threshold for z-score (default: 3.0)
|
|
pub threshold: f64,
|
|
/// Use modified z-score (more robust)
|
|
pub use_modified: bool,
|
|
}
|
|
|
|
/// IQR-based anomaly detector
|
|
#[derive(Debug, Clone)]
|
|
pub struct IQRDetector {
|
|
/// Multiplier for IQR (default: 1.5)
|
|
pub multiplier: f64,
|
|
/// Use Tukey's method for outlier detection
|
|
pub use_tukey_method: bool,
|
|
}
|
|
|
|
/// Isolation Forest anomaly detector
|
|
#[derive(Debug, Clone)]
|
|
pub struct IsolationForestDetector {
|
|
/// Number of trees in the forest
|
|
pub n_trees: usize,
|
|
/// Subsample size for each tree
|
|
pub subsample_size: usize,
|
|
/// Contamination rate (expected proportion of anomalies)
|
|
pub contamination_rate: f64,
|
|
/// Random number generator seed
|
|
pub random_seed: u64,
|
|
/// Trained isolation trees
|
|
pub trees: Vec<IsolationTree>,
|
|
}
|
|
|
|
/// Single isolation tree
|
|
#[derive(Debug, Clone)]
|
|
pub struct IsolationTree {
|
|
/// Root node of the tree
|
|
pub root: Option<TreeNode>,
|
|
/// Maximum depth of the tree
|
|
pub max_depth: usize,
|
|
}
|
|
|
|
/// Node in an isolation tree
|
|
#[derive(Debug, Clone)]
|
|
pub struct TreeNode {
|
|
/// Split feature index
|
|
pub split_feature: usize,
|
|
/// Split threshold
|
|
pub split_threshold: f64,
|
|
/// Left child node
|
|
pub left: Option<Box<TreeNode>>,
|
|
/// Right child node
|
|
pub right: Option<Box<TreeNode>>,
|
|
/// Is this a leaf node?
|
|
pub is_leaf: bool,
|
|
/// Path length to this node
|
|
pub path_length: usize,
|
|
}
|
|
|
|
/// Streaming anomaly detector for real-time detection
|
|
#[derive(Debug, Clone)]
|
|
pub struct StreamingAnomalyDetector {
|
|
/// Detection methods to use
|
|
pub methods: Vec<DetectionMethod>,
|
|
/// Sliding window of recent values
|
|
pub window: Vec<f64>,
|
|
/// Maximum window size
|
|
pub max_window_size: usize,
|
|
/// Update frequency for model retraining
|
|
pub update_frequency: usize,
|
|
/// Current update count
|
|
pub update_count: usize,
|
|
}
|
|
|
|
/// Multivariate anomaly detection result
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct MultivariateAnomalyResult {
|
|
/// Record identifier
|
|
pub record_id: String,
|
|
/// Overall anomaly score
|
|
pub overall_score: f64,
|
|
/// Per-field contributions to anomaly score
|
|
pub field_contributions: HashMap<String, f64>,
|
|
/// Detection method used
|
|
pub detection_method: DetectionMethod,
|
|
/// Is this record anomalous?
|
|
pub is_anomalous: bool,
|
|
/// Confidence of the detection
|
|
pub confidence: f64,
|
|
}
|
|
|
|
impl Default for AnomalyConfig {
|
|
fn default() -> Self {
|
|
Self {
|
|
z_score_threshold: 3.0,
|
|
iqr_multiplier: 1.5,
|
|
contamination_rate: 0.1,
|
|
n_trees: 100,
|
|
subsample_size: 256,
|
|
enable_multivariate: true,
|
|
min_samples: 30,
|
|
enable_streaming: false,
|
|
streaming_window_size: 1000,
|
|
}
|
|
}
|
|
}
|
|
|
|
impl AnomalyDetector {
|
|
/// Create a new anomaly detector with default configuration
|
|
pub fn new() -> Self {
|
|
let config = AnomalyConfig::default();
|
|
Self::with_config(config)
|
|
}
|
|
|
|
/// Create an anomaly detector with custom configuration
|
|
pub fn with_config(config: AnomalyConfig) -> Self {
|
|
Self {
|
|
z_score_detector: ZScoreDetector::new(config.z_score_threshold),
|
|
iqr_detector: IQRDetector::new(config.iqr_multiplier),
|
|
isolation_forest_detector: IsolationForestDetector::new(
|
|
config.n_trees,
|
|
config.subsample_size,
|
|
config.contamination_rate,
|
|
),
|
|
config,
|
|
historical_data: HashMap::new(),
|
|
}
|
|
}
|
|
|
|
/// Detect anomalies in a set of field values
|
|
pub fn detect_anomalies(
|
|
&self,
|
|
fields: &HashMap<String, DataValue>,
|
|
) -> Result<Vec<AnomalyResult>> {
|
|
let mut results = Vec::new();
|
|
|
|
for (field_name, value) in fields {
|
|
if let Some(numeric_value) = value.as_f64() {
|
|
// Get historical data for this field
|
|
if let Some(historical) = self.historical_data.get(field_name) {
|
|
if historical.len() >= self.config.min_samples {
|
|
// Run all detection methods
|
|
if let Some(z_score_result) =
|
|
self.z_score_detector
|
|
.detect(field_name, numeric_value, historical)?
|
|
{
|
|
results.push(z_score_result);
|
|
}
|
|
|
|
if let Some(iqr_result) =
|
|
self.iqr_detector
|
|
.detect(field_name, numeric_value, historical)?
|
|
{
|
|
results.push(iqr_result);
|
|
}
|
|
|
|
if let Some(isolation_result) = self
|
|
.isolation_forest_detector
|
|
.detect_single(field_name, numeric_value, historical)?
|
|
{
|
|
results.push(isolation_result);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
Ok(results)
|
|
}
|
|
|
|
/// Add historical data for a field
|
|
pub fn add_historical_data(&mut self, field_name: String, values: Vec<f64>) {
|
|
self.historical_data.insert(field_name, values);
|
|
}
|
|
|
|
/// Update historical data with new values
|
|
pub fn update_historical_data(&mut self, field_name: &str, value: f64) {
|
|
self.historical_data
|
|
.entry(field_name.to_string())
|
|
.or_insert_with(Vec::new)
|
|
.push(value);
|
|
}
|
|
|
|
/// Train the isolation forest detector
|
|
pub fn train_isolation_forest(&mut self, training_data: &[Vec<f64>]) -> Result<()> {
|
|
self.isolation_forest_detector.train(training_data)?;
|
|
Ok(())
|
|
}
|
|
|
|
/// Detect multivariate anomalies
|
|
pub fn detect_multivariate_anomalies(
|
|
&self,
|
|
records: &[(String, HashMap<String, f64>)],
|
|
) -> Result<Vec<MultivariateAnomalyResult>> {
|
|
if !self.config.enable_multivariate {
|
|
return Ok(vec![]);
|
|
}
|
|
|
|
let mut results = Vec::new();
|
|
|
|
// Extract feature matrix
|
|
let feature_names: Vec<String> = records
|
|
.first()
|
|
.map(|(_, fields)| fields.keys().cloned().collect())
|
|
.unwrap_or_default();
|
|
|
|
for (record_id, fields) in records {
|
|
let feature_vector: Vec<f64> = feature_names
|
|
.iter()
|
|
.map(|name| fields.get(name).copied().unwrap_or(0.0))
|
|
.collect();
|
|
|
|
// Use isolation forest for multivariate detection
|
|
let anomaly_score = self
|
|
.isolation_forest_detector
|
|
.score_multivariate(&feature_vector)?;
|
|
let is_anomalous =
|
|
anomaly_score > self.isolation_forest_detector.contamination_threshold();
|
|
|
|
// Calculate per-field contributions (simplified)
|
|
let mut field_contributions = HashMap::new();
|
|
for (i, name) in feature_names.iter().enumerate() {
|
|
if let Some(value) = feature_vector.get(i) {
|
|
// Simplified contribution calculation
|
|
field_contributions.insert(name.clone(), value.abs());
|
|
}
|
|
}
|
|
|
|
results.push(MultivariateAnomalyResult {
|
|
record_id: record_id.clone(),
|
|
overall_score: anomaly_score,
|
|
field_contributions,
|
|
detection_method: DetectionMethod::IsolationForest,
|
|
is_anomalous,
|
|
confidence: if is_anomalous { 0.8 } else { 0.6 },
|
|
});
|
|
}
|
|
|
|
Ok(results)
|
|
}
|
|
}
|
|
|
|
impl Default for AnomalyDetector {
|
|
fn default() -> Self {
|
|
Self::new()
|
|
}
|
|
}
|
|
|
|
impl ZScoreDetector {
|
|
/// Create a new Z-score detector
|
|
pub fn new(threshold: f64) -> Self {
|
|
Self {
|
|
threshold,
|
|
use_modified: false,
|
|
}
|
|
}
|
|
|
|
/// Create a new modified Z-score detector (more robust)
|
|
pub fn new_modified(threshold: f64) -> Self {
|
|
Self {
|
|
threshold,
|
|
use_modified: true,
|
|
}
|
|
}
|
|
|
|
/// Detect anomalies using Z-score method
|
|
pub fn detect(
|
|
&self,
|
|
field_name: &str,
|
|
value: f64,
|
|
historical_data: &[f64],
|
|
) -> Result<Option<AnomalyResult>> {
|
|
if historical_data.len() < 3 {
|
|
return Ok(None);
|
|
}
|
|
|
|
let (score, context) = if self.use_modified {
|
|
self.calculate_modified_z_score(value, historical_data)?
|
|
} else {
|
|
self.calculate_z_score(value, historical_data)?
|
|
};
|
|
|
|
if score.abs() > self.threshold {
|
|
let severity = match score.abs() {
|
|
s if s > 4.0 => AnomalySeverity::Critical,
|
|
s if s > 3.5 => AnomalySeverity::High,
|
|
s if s > 3.0 => AnomalySeverity::Medium,
|
|
_ => AnomalySeverity::Low,
|
|
};
|
|
|
|
Ok(Some(AnomalyResult {
|
|
field_name: field_name.to_string(),
|
|
anomalous_value: value.to_string(),
|
|
anomaly_score: score.abs(),
|
|
detection_method: if self.use_modified {
|
|
DetectionMethod::ModifiedZScore
|
|
} else {
|
|
DetectionMethod::ZScore
|
|
},
|
|
threshold: self.threshold,
|
|
context,
|
|
severity,
|
|
confidence: (score.abs() / self.threshold).min(1.0),
|
|
}))
|
|
} else {
|
|
Ok(None)
|
|
}
|
|
}
|
|
|
|
fn calculate_z_score(&self, value: f64, data: &[f64]) -> Result<(f64, AnomalyContext)> {
|
|
let mean = data.iter().sum::<f64>() / data.len() as f64;
|
|
let variance =
|
|
data.iter().map(|x| (x - mean).powi(2)).sum::<f64>() / (data.len() - 1) as f64;
|
|
let std_dev = variance.sqrt();
|
|
|
|
if std_dev == 0.0 {
|
|
return Ok((0.0, self.build_context(data, mean, std_dev)));
|
|
}
|
|
|
|
let z_score = (value - mean) / std_dev;
|
|
Ok((z_score, self.build_context(data, mean, std_dev)))
|
|
}
|
|
|
|
fn calculate_modified_z_score(
|
|
&self,
|
|
value: f64,
|
|
data: &[f64],
|
|
) -> Result<(f64, AnomalyContext)> {
|
|
let mut sorted_data = data.to_vec();
|
|
sorted_data.sort_by(|a, b| a.total_cmp(b));
|
|
|
|
let median = sorted_data.median();
|
|
|
|
// Calculate median absolute deviation (MAD)
|
|
let deviations: Vec<f64> = data.iter().map(|&x| (x - median).abs()).collect();
|
|
let mut sorted_deviations = deviations;
|
|
sorted_deviations.sort_by(|a, b| a.total_cmp(b));
|
|
let mad = sorted_deviations.median();
|
|
|
|
if mad == 0.0 {
|
|
return Ok((0.0, self.build_context(data, median, 0.0)));
|
|
}
|
|
|
|
// Modified Z-score formula: M_i = 0.6745 * (x_i - median) / MAD
|
|
let modified_z_score = 0.6745 * (value - median) / mad;
|
|
Ok((
|
|
modified_z_score,
|
|
self.build_context(data, median, mad * 1.4826),
|
|
)) // Convert MAD to std dev equivalent
|
|
}
|
|
|
|
fn build_context(&self, data: &[f64], mean: f64, std_dev: f64) -> AnomalyContext {
|
|
let mut sorted_data = data.to_vec();
|
|
sorted_data.sort_by(|a, b| a.total_cmp(b));
|
|
|
|
AnomalyContext {
|
|
mean,
|
|
std_dev,
|
|
median: sorted_data.median(),
|
|
q1: sorted_data.quantile(0.25),
|
|
q3: sorted_data.quantile(0.75),
|
|
iqr: sorted_data.quantile(0.75) - sorted_data.quantile(0.25),
|
|
min: sorted_data.first().copied().unwrap_or(0.0),
|
|
max: sorted_data.last().copied().unwrap_or(0.0),
|
|
sample_size: data.len(),
|
|
}
|
|
}
|
|
}
|
|
|
|
impl IQRDetector {
|
|
/// Create a new IQR detector
|
|
pub fn new(multiplier: f64) -> Self {
|
|
Self {
|
|
multiplier,
|
|
use_tukey_method: true,
|
|
}
|
|
}
|
|
|
|
/// Detect anomalies using IQR method
|
|
pub fn detect(
|
|
&self,
|
|
field_name: &str,
|
|
value: f64,
|
|
historical_data: &[f64],
|
|
) -> Result<Option<AnomalyResult>> {
|
|
if historical_data.len() < 4 {
|
|
return Ok(None);
|
|
}
|
|
|
|
let mut sorted_data = historical_data.to_vec();
|
|
sorted_data.sort_by(|a, b| a.total_cmp(b));
|
|
|
|
let q1 = sorted_data.quantile(0.25);
|
|
let q3 = sorted_data.quantile(0.75);
|
|
let iqr = q3 - q1;
|
|
|
|
if iqr == 0.0 {
|
|
return Ok(None);
|
|
}
|
|
|
|
let lower_bound = q1 - self.multiplier * iqr;
|
|
let upper_bound = q3 + self.multiplier * iqr;
|
|
|
|
if value < lower_bound || value > upper_bound {
|
|
let distance_from_bound = if value < lower_bound {
|
|
(lower_bound - value) / iqr
|
|
} else {
|
|
(value - upper_bound) / iqr
|
|
};
|
|
|
|
let severity = match distance_from_bound {
|
|
d if d > 3.0 => AnomalySeverity::Critical,
|
|
d if d > 2.0 => AnomalySeverity::High,
|
|
d if d > 1.0 => AnomalySeverity::Medium,
|
|
_ => AnomalySeverity::Low,
|
|
};
|
|
|
|
let context = AnomalyContext {
|
|
mean: historical_data.iter().sum::<f64>() / historical_data.len() as f64,
|
|
std_dev: {
|
|
let mean = historical_data.iter().sum::<f64>() / historical_data.len() as f64;
|
|
let variance = historical_data
|
|
.iter()
|
|
.map(|x| (x - mean).powi(2))
|
|
.sum::<f64>()
|
|
/ (historical_data.len() - 1) as f64;
|
|
variance.sqrt()
|
|
},
|
|
median: sorted_data.median(),
|
|
q1,
|
|
q3,
|
|
iqr,
|
|
min: sorted_data.first().copied().unwrap_or(0.0),
|
|
max: sorted_data.last().copied().unwrap_or(0.0),
|
|
sample_size: historical_data.len(),
|
|
};
|
|
|
|
Ok(Some(AnomalyResult {
|
|
field_name: field_name.to_string(),
|
|
anomalous_value: value.to_string(),
|
|
anomaly_score: distance_from_bound,
|
|
detection_method: DetectionMethod::IQR,
|
|
threshold: self.multiplier,
|
|
context,
|
|
severity,
|
|
confidence: (distance_from_bound / 2.0).min(1.0),
|
|
}))
|
|
} else {
|
|
Ok(None)
|
|
}
|
|
}
|
|
}
|
|
|
|
impl IsolationForestDetector {
|
|
/// Create a new isolation forest detector
|
|
pub fn new(n_trees: usize, subsample_size: usize, contamination_rate: f64) -> Self {
|
|
Self {
|
|
n_trees,
|
|
subsample_size,
|
|
contamination_rate,
|
|
random_seed: 42,
|
|
trees: Vec::new(),
|
|
}
|
|
}
|
|
|
|
/// Train the isolation forest
|
|
pub fn train(&mut self, training_data: &[Vec<f64>]) -> Result<()> {
|
|
if training_data.is_empty() {
|
|
return Err(ValidationError::Anomaly("Empty training data".to_string()));
|
|
}
|
|
|
|
let mut rng = StdRng::seed_from_u64(self.random_seed);
|
|
self.trees.clear();
|
|
|
|
let n_features = training_data[0].len();
|
|
let max_depth = (self.subsample_size as f64).log2().ceil() as usize;
|
|
|
|
for _ in 0..self.n_trees {
|
|
// Sample data for this tree
|
|
let mut sampled_data = Vec::new();
|
|
let sample_size = self.subsample_size.min(training_data.len());
|
|
|
|
for _ in 0..sample_size {
|
|
let idx = rng.gen_range(0..training_data.len());
|
|
sampled_data.push(training_data[idx].clone());
|
|
}
|
|
|
|
// Build isolation tree
|
|
let tree = self.build_tree(&sampled_data, 0, max_depth, n_features, &mut rng);
|
|
self.trees.push(tree);
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
/// Detect anomaly for a single value
|
|
pub fn detect_single(
|
|
&self,
|
|
field_name: &str,
|
|
value: f64,
|
|
_historical_data: &[f64],
|
|
) -> Result<Option<AnomalyResult>> {
|
|
if self.trees.is_empty() {
|
|
return Ok(None);
|
|
}
|
|
|
|
// For single-value detection, create a simple 1D point
|
|
let point = vec![value];
|
|
let anomaly_score = self.score_multivariate(&point)?;
|
|
let threshold = self.contamination_threshold();
|
|
|
|
if anomaly_score > threshold {
|
|
let severity = match anomaly_score {
|
|
s if s > 0.8 => AnomalySeverity::Critical,
|
|
s if s > 0.7 => AnomalySeverity::High,
|
|
s if s > 0.6 => AnomalySeverity::Medium,
|
|
_ => AnomalySeverity::Low,
|
|
};
|
|
|
|
// Build minimal context for single value
|
|
let context = AnomalyContext {
|
|
mean: value,
|
|
std_dev: 0.0,
|
|
median: value,
|
|
q1: value,
|
|
q3: value,
|
|
iqr: 0.0,
|
|
min: value,
|
|
max: value,
|
|
sample_size: 1,
|
|
};
|
|
|
|
Ok(Some(AnomalyResult {
|
|
field_name: field_name.to_string(),
|
|
anomalous_value: value.to_string(),
|
|
anomaly_score,
|
|
detection_method: DetectionMethod::IsolationForest,
|
|
threshold,
|
|
context,
|
|
severity,
|
|
confidence: anomaly_score,
|
|
}))
|
|
} else {
|
|
Ok(None)
|
|
}
|
|
}
|
|
|
|
/// Score a multivariate point
|
|
pub fn score_multivariate(&self, point: &[f64]) -> Result<f64> {
|
|
if self.trees.is_empty() {
|
|
return Ok(0.0);
|
|
}
|
|
|
|
let mut total_path_length = 0.0;
|
|
for tree in &self.trees {
|
|
let path_length = self.path_length(tree, point, 0);
|
|
total_path_length += path_length;
|
|
}
|
|
|
|
let avg_path_length = total_path_length / self.trees.len() as f64;
|
|
|
|
// Convert to anomaly score using the standard isolation forest formula
|
|
let c_n = self.average_path_length(self.subsample_size);
|
|
let anomaly_score = (-avg_path_length / c_n).exp2();
|
|
|
|
Ok(anomaly_score)
|
|
}
|
|
|
|
/// Get contamination threshold
|
|
pub fn contamination_threshold(&self) -> f64 {
|
|
// Convert contamination rate to threshold
|
|
// Higher contamination rate = lower threshold for flagging anomalies
|
|
1.0 - self.contamination_rate
|
|
}
|
|
|
|
fn build_tree(
|
|
&self,
|
|
data: &[Vec<f64>],
|
|
depth: usize,
|
|
max_depth: usize,
|
|
n_features: usize,
|
|
rng: &mut StdRng,
|
|
) -> IsolationTree {
|
|
if data.is_empty() || depth >= max_depth || data.len() <= 1 {
|
|
return IsolationTree {
|
|
root: None,
|
|
max_depth,
|
|
};
|
|
}
|
|
|
|
// Randomly select feature to split on
|
|
let split_feature = rng.gen_range(0..n_features);
|
|
|
|
// Find min and max values for the selected feature
|
|
let mut min_val = f64::INFINITY;
|
|
let mut max_val = f64::NEG_INFINITY;
|
|
|
|
for point in data {
|
|
if let Some(&val) = point.get(split_feature) {
|
|
min_val = min_val.min(val);
|
|
max_val = max_val.max(val);
|
|
}
|
|
}
|
|
|
|
if min_val >= max_val {
|
|
return IsolationTree {
|
|
root: None,
|
|
max_depth,
|
|
};
|
|
}
|
|
|
|
// Random split threshold
|
|
let split_threshold = rng.gen_range(min_val..max_val);
|
|
|
|
// Split data
|
|
let mut left_data = Vec::new();
|
|
let mut right_data = Vec::new();
|
|
|
|
for point in data {
|
|
if let Some(&val) = point.get(split_feature) {
|
|
if val < split_threshold {
|
|
left_data.push(point.clone());
|
|
} else {
|
|
right_data.push(point.clone());
|
|
}
|
|
}
|
|
}
|
|
|
|
// Recursively build subtrees
|
|
let left_tree = if !left_data.is_empty() {
|
|
Some(Box::new(self.build_tree_node(
|
|
&left_data,
|
|
depth + 1,
|
|
max_depth,
|
|
n_features,
|
|
rng,
|
|
)))
|
|
} else {
|
|
None
|
|
};
|
|
|
|
let right_tree = if !right_data.is_empty() {
|
|
Some(Box::new(self.build_tree_node(
|
|
&right_data,
|
|
depth + 1,
|
|
max_depth,
|
|
n_features,
|
|
rng,
|
|
)))
|
|
} else {
|
|
None
|
|
};
|
|
|
|
IsolationTree {
|
|
root: Some(TreeNode {
|
|
split_feature,
|
|
split_threshold,
|
|
left: left_tree,
|
|
right: right_tree,
|
|
is_leaf: false,
|
|
path_length: depth,
|
|
}),
|
|
max_depth,
|
|
}
|
|
}
|
|
|
|
fn build_tree_node(
|
|
&self,
|
|
data: &[Vec<f64>],
|
|
depth: usize,
|
|
max_depth: usize,
|
|
n_features: usize,
|
|
rng: &mut StdRng,
|
|
) -> TreeNode {
|
|
if data.is_empty() || depth >= max_depth || data.len() <= 1 {
|
|
return TreeNode {
|
|
split_feature: 0,
|
|
split_threshold: 0.0,
|
|
left: None,
|
|
right: None,
|
|
is_leaf: true,
|
|
path_length: depth,
|
|
};
|
|
}
|
|
|
|
// Same logic as build_tree but returns TreeNode
|
|
let split_feature = rng.gen_range(0..n_features);
|
|
|
|
let mut min_val = f64::INFINITY;
|
|
let mut max_val = f64::NEG_INFINITY;
|
|
|
|
for point in data {
|
|
if let Some(&val) = point.get(split_feature) {
|
|
min_val = min_val.min(val);
|
|
max_val = max_val.max(val);
|
|
}
|
|
}
|
|
|
|
if min_val >= max_val {
|
|
return TreeNode {
|
|
split_feature,
|
|
split_threshold: min_val,
|
|
left: None,
|
|
right: None,
|
|
is_leaf: true,
|
|
path_length: depth,
|
|
};
|
|
}
|
|
|
|
let split_threshold = rng.gen_range(min_val..max_val);
|
|
|
|
let mut left_data = Vec::new();
|
|
let mut right_data = Vec::new();
|
|
|
|
for point in data {
|
|
if let Some(&val) = point.get(split_feature) {
|
|
if val < split_threshold {
|
|
left_data.push(point.clone());
|
|
} else {
|
|
right_data.push(point.clone());
|
|
}
|
|
}
|
|
}
|
|
|
|
let left_child = if !left_data.is_empty() {
|
|
Some(Box::new(self.build_tree_node(
|
|
&left_data,
|
|
depth + 1,
|
|
max_depth,
|
|
n_features,
|
|
rng,
|
|
)))
|
|
} else {
|
|
None
|
|
};
|
|
|
|
let right_child = if !right_data.is_empty() {
|
|
Some(Box::new(self.build_tree_node(
|
|
&right_data,
|
|
depth + 1,
|
|
max_depth,
|
|
n_features,
|
|
rng,
|
|
)))
|
|
} else {
|
|
None
|
|
};
|
|
|
|
let is_leaf = left_child.is_none() && right_child.is_none();
|
|
|
|
TreeNode {
|
|
split_feature,
|
|
split_threshold,
|
|
left: left_child,
|
|
right: right_child,
|
|
is_leaf,
|
|
path_length: depth,
|
|
}
|
|
}
|
|
|
|
fn path_length(&self, tree: &IsolationTree, point: &[f64], current_depth: usize) -> f64 {
|
|
if let Some(ref root) = tree.root {
|
|
self.path_length_recursive(root, point, current_depth)
|
|
} else {
|
|
current_depth as f64
|
|
}
|
|
}
|
|
|
|
fn path_length_recursive(&self, node: &TreeNode, point: &[f64], current_depth: usize) -> f64 {
|
|
if node.is_leaf || node.left.is_none() && node.right.is_none() {
|
|
return current_depth as f64 + self.average_path_length(1);
|
|
}
|
|
|
|
if let Some(value) = point.get(node.split_feature) {
|
|
if *value < node.split_threshold {
|
|
if let Some(ref left) = node.left {
|
|
return self.path_length_recursive(left, point, current_depth + 1);
|
|
}
|
|
} else if let Some(ref right) = node.right {
|
|
return self.path_length_recursive(right, point, current_depth + 1);
|
|
}
|
|
}
|
|
|
|
current_depth as f64
|
|
}
|
|
|
|
fn average_path_length(&self, n: usize) -> f64 {
|
|
if n <= 1 {
|
|
return 0.0;
|
|
}
|
|
|
|
// Average path length of unsuccessful search in BST
|
|
2.0 * ((n - 1) as f64).ln() - 2.0 * (n - 1) as f64 / n as f64
|
|
}
|
|
}
|
|
|
|
impl StreamingAnomalyDetector {
|
|
/// Create a new streaming detector
|
|
pub fn new(methods: Vec<DetectionMethod>, max_window_size: usize) -> Self {
|
|
Self {
|
|
methods,
|
|
window: Vec::new(),
|
|
max_window_size,
|
|
update_frequency: 100,
|
|
update_count: 0,
|
|
}
|
|
}
|
|
|
|
/// Add a new value and check for anomalies
|
|
pub fn add_and_detect(&mut self, value: f64) -> Result<Vec<AnomalyResult>> {
|
|
// Add value to window
|
|
self.window.push(value);
|
|
|
|
// Maintain window size
|
|
if self.window.len() > self.max_window_size {
|
|
self.window.remove(0);
|
|
}
|
|
|
|
self.update_count += 1;
|
|
|
|
// Detect anomalies if we have enough data
|
|
if self.window.len() >= 30 {
|
|
let mut results = Vec::new();
|
|
|
|
for method in &self.methods {
|
|
match method {
|
|
DetectionMethod::ZScore => {
|
|
let detector = ZScoreDetector::new(3.0);
|
|
if let Some(result) = detector.detect("streaming", value, &self.window)? {
|
|
results.push(result);
|
|
}
|
|
}
|
|
DetectionMethod::IQR => {
|
|
let detector = IQRDetector::new(1.5);
|
|
if let Some(result) = detector.detect("streaming", value, &self.window)? {
|
|
results.push(result);
|
|
}
|
|
}
|
|
_ => {} // Other methods would be implemented similarly
|
|
}
|
|
}
|
|
|
|
return Ok(results);
|
|
}
|
|
|
|
Ok(vec![])
|
|
}
|
|
|
|
/// Get current window statistics
|
|
pub fn window_stats(&self) -> Option<AnomalyContext> {
|
|
if self.window.len() < 2 {
|
|
return None;
|
|
}
|
|
|
|
let mut sorted = self.window.clone();
|
|
sorted.sort_by(|a, b| a.total_cmp(b));
|
|
|
|
Some(AnomalyContext {
|
|
mean: self.window.iter().sum::<f64>() / self.window.len() as f64,
|
|
std_dev: {
|
|
let mean = self.window.iter().sum::<f64>() / self.window.len() as f64;
|
|
let variance = self.window.iter().map(|x| (x - mean).powi(2)).sum::<f64>()
|
|
/ (self.window.len() - 1) as f64;
|
|
variance.sqrt()
|
|
},
|
|
median: sorted.median(),
|
|
q1: sorted.quantile(0.25),
|
|
q3: sorted.quantile(0.75),
|
|
iqr: sorted.quantile(0.75) - sorted.quantile(0.25),
|
|
min: sorted.first().copied().unwrap_or(0.0),
|
|
max: sorted.last().copied().unwrap_or(0.0),
|
|
sample_size: self.window.len(),
|
|
})
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn test_anomaly_detector_creation() {
|
|
let detector = AnomalyDetector::new();
|
|
assert_eq!(detector.config.z_score_threshold, 3.0);
|
|
assert_eq!(detector.config.iqr_multiplier, 1.5);
|
|
}
|
|
|
|
#[test]
|
|
fn test_z_score_detection() {
|
|
let detector = ZScoreDetector::new(3.0);
|
|
let data = vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0];
|
|
|
|
// Normal value
|
|
let result = detector.detect("test", 5.5, &data).unwrap();
|
|
assert!(result.is_none());
|
|
|
|
// Anomalous value
|
|
let result = detector.detect("test", 50.0, &data).unwrap();
|
|
assert!(result.is_some());
|
|
|
|
if let Some(anomaly) = result {
|
|
assert!(anomaly.anomaly_score > 3.0);
|
|
assert_eq!(anomaly.detection_method, DetectionMethod::ZScore);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn test_iqr_detection() {
|
|
let detector = IQRDetector::new(1.5);
|
|
let data = vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0];
|
|
|
|
// Normal value
|
|
let result = detector.detect("test", 5.5, &data).unwrap();
|
|
assert!(result.is_none());
|
|
|
|
// Anomalous value
|
|
let result = detector.detect("test", 50.0, &data).unwrap();
|
|
assert!(result.is_some());
|
|
|
|
if let Some(anomaly) = result {
|
|
assert!(anomaly.anomaly_score > 0.0);
|
|
assert_eq!(anomaly.detection_method, DetectionMethod::IQR);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn test_isolation_forest() {
|
|
let mut detector = IsolationForestDetector::new(10, 8, 0.1);
|
|
|
|
// Train with normal data
|
|
let training_data = vec![
|
|
vec![1.0],
|
|
vec![2.0],
|
|
vec![3.0],
|
|
vec![4.0],
|
|
vec![5.0],
|
|
vec![6.0],
|
|
vec![7.0],
|
|
vec![8.0],
|
|
vec![9.0],
|
|
vec![10.0],
|
|
];
|
|
|
|
detector.train(&training_data).unwrap();
|
|
|
|
// Test normal value
|
|
let normal_score = detector.score_multivariate(&[5.0]).unwrap();
|
|
|
|
// Test anomalous value
|
|
let anomaly_score = detector.score_multivariate(&[100.0]).unwrap();
|
|
|
|
// Anomaly should have higher score
|
|
assert!(anomaly_score >= normal_score);
|
|
}
|
|
|
|
#[test]
|
|
fn test_streaming_detector() {
|
|
let mut detector =
|
|
StreamingAnomalyDetector::new(vec![DetectionMethod::ZScore, DetectionMethod::IQR], 100);
|
|
|
|
// Add normal values
|
|
for i in 1..=30 {
|
|
let results = detector.add_and_detect(i as f64).unwrap();
|
|
if i == 30 {
|
|
assert!(results.is_empty()); // No anomalies in normal sequence
|
|
}
|
|
}
|
|
|
|
// Add anomalous value
|
|
let results = detector.add_and_detect(1000.0).unwrap();
|
|
assert!(!results.is_empty()); // Should detect anomaly
|
|
}
|
|
|
|
#[test]
|
|
fn test_anomaly_severity() {
|
|
use std::cmp::Ordering;
|
|
|
|
assert_eq!(
|
|
AnomalySeverity::Low.cmp(&AnomalySeverity::Medium),
|
|
Ordering::Less
|
|
);
|
|
assert_eq!(
|
|
AnomalySeverity::Medium.cmp(&AnomalySeverity::High),
|
|
Ordering::Less
|
|
);
|
|
assert_eq!(
|
|
AnomalySeverity::High.cmp(&AnomalySeverity::Critical),
|
|
Ordering::Less
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn test_modified_z_score() {
|
|
let detector = ZScoreDetector::new_modified(3.5);
|
|
let data = vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 100.0]; // Data with outlier
|
|
|
|
let result = detector.detect("test", 100.0, &data).unwrap();
|
|
|
|
if let Some(anomaly) = result {
|
|
assert_eq!(anomaly.detection_method, DetectionMethod::ModifiedZScore);
|
|
assert!(anomaly.anomaly_score > 0.0);
|
|
}
|
|
}
|
|
}
|