style: cargo fmt --workspace (whitespace/wrapping only, no semantic change)
Whole-workspace rustfmt pass picked up while iterating on Mamba GPU backward work. Verified formatting-only via diff sampling; no logic changed. Co-Authored-By: Claude Sonnet 5 <[email protected]>
This commit is contained in:
osobh
@@ -5,8 +5,8 @@
|
||||
|
||||
use crate::prelude::*;
|
||||
use crate::validation_framework::*;
|
||||
use std::collections::HashMap;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
|
||||
/// Regression test configuration
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
@@ -179,23 +179,26 @@ impl RegressionTestSuite {
|
||||
let current_report = self.validation_framework.run_comprehensive_validation()?;
|
||||
|
||||
// Analyze regressions
|
||||
let performance_regressions = self.analyze_performance_regressions(&baseline_report, ¤t_report);
|
||||
let accuracy_regressions = self.analyze_accuracy_regressions(&baseline_report, ¤t_report);
|
||||
let feature_regressions = self.analyze_feature_regressions(&baseline_report, ¤t_report);
|
||||
let performance_regressions =
|
||||
self.analyze_performance_regressions(&baseline_report, ¤t_report);
|
||||
let accuracy_regressions =
|
||||
self.analyze_accuracy_regressions(&baseline_report, ¤t_report);
|
||||
let feature_regressions =
|
||||
self.analyze_feature_regressions(&baseline_report, ¤t_report);
|
||||
|
||||
// Detect new issues
|
||||
let new_issues = self.detect_new_issues(&baseline_report, ¤t_report);
|
||||
|
||||
// Generate summary
|
||||
let summary = self.generate_regression_summary(
|
||||
&performance_regressions,
|
||||
&accuracy_regressions,
|
||||
&feature_regressions,
|
||||
&new_issues,
|
||||
&performance_regressions,
|
||||
&accuracy_regressions,
|
||||
&feature_regressions,
|
||||
&new_issues,
|
||||
);
|
||||
|
||||
let all_tests_passed = summary.critical_regressions == 0 &&
|
||||
(self.config.strict_mode == false || summary.total_regressions == 0);
|
||||
let all_tests_passed = summary.critical_regressions == 0
|
||||
&& (self.config.strict_mode == false || summary.total_regressions == 0);
|
||||
|
||||
let result = RegressionTestResult {
|
||||
timestamp: chrono::Utc::now(),
|
||||
@@ -209,10 +212,10 @@ impl RegressionTestSuite {
|
||||
|
||||
let elapsed = start_time.elapsed();
|
||||
tracing::info!(
|
||||
"Regression test suite completed in {:.2}s: {} total regressions, {} critical",
|
||||
elapsed.as_secs_f64(),
|
||||
result.summary.total_regressions,
|
||||
result.summary.critical_regressions
|
||||
"Regression test suite completed in {:.2}s: {} total regressions, {} critical",
|
||||
elapsed.as_secs_f64(),
|
||||
result.summary.total_regressions,
|
||||
result.summary.critical_regressions
|
||||
);
|
||||
|
||||
Ok(result)
|
||||
@@ -222,81 +225,93 @@ impl RegressionTestSuite {
|
||||
fn load_baseline_report(&mut self) -> Result<ValidationReport> {
|
||||
if std::path::Path::new(&self.config.baseline_report_path).exists() {
|
||||
ValidationFramework::load_report(&self.config.baseline_report_path)
|
||||
} else {
|
||||
// Generate baseline report if none exists
|
||||
tracing::warn!("No baseline report found, generating new baseline");
|
||||
let report = self.validation_framework.run_comprehensive_validation()?;
|
||||
} else {
|
||||
// Generate baseline report if none exists
|
||||
tracing::warn!("No baseline report found, generating new baseline");
|
||||
let report = self.validation_framework.run_comprehensive_validation()?;
|
||||
|
||||
// Save as baseline
|
||||
self.validation_framework.save_report(&report, &self.config.baseline_report_path)?;
|
||||
Ok(report)
|
||||
}
|
||||
// Save as baseline
|
||||
self.validation_framework
|
||||
.save_report(&report, &self.config.baseline_report_path)?;
|
||||
Ok(report)
|
||||
}
|
||||
}
|
||||
|
||||
/// Analyze performance regressions
|
||||
fn analyze_performance_regressions(
|
||||
/// Analyze performance regressions
|
||||
fn analyze_performance_regressions(
|
||||
&self,
|
||||
baseline: &ValidationReport,
|
||||
current: &ValidationReport,
|
||||
) -> Vec<PerformanceRegression> {
|
||||
let mut regressions = Vec::new();
|
||||
) -> Vec<PerformanceRegression> {
|
||||
let mut regressions = Vec::new();
|
||||
|
||||
// Match performance results by test name
|
||||
for current_result in ¤t.performance_results {
|
||||
if let Some(baseline_result) = baseline.performance_results.iter()
|
||||
// Match performance results by test name
|
||||
for current_result in ¤t.performance_results {
|
||||
if let Some(baseline_result) = baseline
|
||||
.performance_results
|
||||
.iter()
|
||||
.find(|r| r.test_name == current_result.test_name)
|
||||
{
|
||||
let regression_percentage = (current_result.actual_performance - baseline_result.actual_performance)
|
||||
{
|
||||
let regression_percentage = (current_result.actual_performance
|
||||
- baseline_result.actual_performance)
|
||||
/ baseline_result.actual_performance;
|
||||
|
||||
// Check if this is a significant regression
|
||||
if regression_percentage < -self.config.max_performance_regression {
|
||||
let is_critical = regression_percentage < -0.20; // 20% regression is critical
|
||||
// Check if this is a significant regression
|
||||
if regression_percentage < -self.config.max_performance_regression {
|
||||
let is_critical = regression_percentage < -0.20; // 20% regression is critical
|
||||
|
||||
let mut context = HashMap::new();
|
||||
context.insert("baseline_performance".to_string(),
|
||||
baseline_result.actual_performance.to_string());
|
||||
context.insert("current_performance".to_string(),
|
||||
current_result.actual_performance.to_string());
|
||||
context.insert("expected_performance".to_string(),
|
||||
current_result.expected_performance.to_string());
|
||||
let mut context = HashMap::new();
|
||||
context.insert(
|
||||
"baseline_performance".to_string(),
|
||||
baseline_result.actual_performance.to_string(),
|
||||
);
|
||||
context.insert(
|
||||
"current_performance".to_string(),
|
||||
current_result.actual_performance.to_string(),
|
||||
);
|
||||
context.insert(
|
||||
"expected_performance".to_string(),
|
||||
current_result.expected_performance.to_string(),
|
||||
);
|
||||
|
||||
regressions.push(PerformanceRegression {
|
||||
test_name: current_result.test_name.clone(),
|
||||
baseline_performance: baseline_result.actual_performance,
|
||||
current_performance: current_result.actual_performance,
|
||||
regression_percentage: regression_percentage * 100.0, // Convert to percentage
|
||||
is_critical,
|
||||
context,
|
||||
});
|
||||
}
|
||||
regressions.push(PerformanceRegression {
|
||||
test_name: current_result.test_name.clone(),
|
||||
baseline_performance: baseline_result.actual_performance,
|
||||
current_performance: current_result.actual_performance,
|
||||
regression_percentage: regression_percentage * 100.0, // Convert to percentage
|
||||
is_critical,
|
||||
context,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
regressions
|
||||
}
|
||||
|
||||
/// Analyze accuracy regressions
|
||||
fn analyze_accuracy_regressions(
|
||||
regressions
|
||||
}
|
||||
|
||||
/// Analyze accuracy regressions
|
||||
fn analyze_accuracy_regressions(
|
||||
&self,
|
||||
baseline: &ValidationReport,
|
||||
current: &ValidationReport,
|
||||
) -> Vec<AccuracyRegression> {
|
||||
let mut regressions = Vec::new();
|
||||
) -> Vec<AccuracyRegression> {
|
||||
let mut regressions = Vec::new();
|
||||
|
||||
for current_result in ¤t.accuracy_results {
|
||||
if let Some(baseline_result) = baseline.accuracy_results.iter()
|
||||
for current_result in ¤t.accuracy_results {
|
||||
if let Some(baseline_result) = baseline
|
||||
.accuracy_results
|
||||
.iter()
|
||||
.find(|r| r.test_name == current_result.test_name)
|
||||
{
|
||||
// Check if accuracy degraded
|
||||
let max_error_increase = current_result.max_error - baseline_result.max_error;
|
||||
let mean_error_increase = current_result.mean_error - baseline_result.mean_error;
|
||||
{
|
||||
// Check if accuracy degraded
|
||||
let max_error_increase = current_result.max_error - baseline_result.max_error;
|
||||
let mean_error_increase = current_result.mean_error - baseline_result.mean_error;
|
||||
|
||||
let accuracy_degraded = max_error_increase > self.config.max_accuracy_degradation ||
|
||||
mean_error_increase > self.config.max_accuracy_degradation;
|
||||
let accuracy_degraded = max_error_increase > self.config.max_accuracy_degradation
|
||||
|| mean_error_increase > self.config.max_accuracy_degradation;
|
||||
|
||||
if accuracy_degraded {
|
||||
let degradation_details = format!(
|
||||
if accuracy_degraded {
|
||||
let degradation_details = format!(
|
||||
"Max error: {:.2e} -> {:.2e} (+{:.2e}), Mean error: {:.2e} -> {:.2e} (+{:.2e})",
|
||||
baseline_result.max_error,
|
||||
current_result.max_error,
|
||||
@@ -304,425 +319,438 @@ impl RegressionTestSuite {
|
||||
baseline_result.mean_error,
|
||||
current_result.mean_error,
|
||||
mean_error_increase
|
||||
);
|
||||
);
|
||||
|
||||
regressions.push(AccuracyRegression {
|
||||
test_name: current_result.test_name.clone(),
|
||||
baseline_max_error: baseline_result.max_error,
|
||||
baseline_mean_error: baseline_result.mean_error,
|
||||
current_max_error: current_result.max_error,
|
||||
current_mean_error: current_result.mean_error,
|
||||
accuracy_degraded: true,
|
||||
degradation_details,
|
||||
});
|
||||
}
|
||||
regressions.push(AccuracyRegression {
|
||||
test_name: current_result.test_name.clone(),
|
||||
baseline_max_error: baseline_result.max_error,
|
||||
baseline_mean_error: baseline_result.mean_error,
|
||||
current_max_error: current_result.max_error,
|
||||
current_mean_error: current_result.mean_error,
|
||||
accuracy_degraded: true,
|
||||
degradation_details,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
regressions
|
||||
}
|
||||
|
||||
/// Analyze feature regressions
|
||||
fn analyze_feature_regressions(
|
||||
regressions
|
||||
}
|
||||
|
||||
/// Analyze feature regressions
|
||||
fn analyze_feature_regressions(
|
||||
&self,
|
||||
baseline: &ValidationReport,
|
||||
current: &ValidationReport,
|
||||
) -> Vec<FeatureRegression> {
|
||||
let mut regressions = Vec::new();
|
||||
) -> Vec<FeatureRegression> {
|
||||
let mut regressions = Vec::new();
|
||||
|
||||
for current_feature in ¤t.feature_results {
|
||||
if let Some(baseline_feature) = baseline.feature_results.iter()
|
||||
for current_feature in ¤t.feature_results {
|
||||
if let Some(baseline_feature) = baseline
|
||||
.feature_results
|
||||
.iter()
|
||||
.find(|f| f.feature_name == current_feature.feature_name)
|
||||
{
|
||||
let baseline_status = FeatureStatus {
|
||||
implemented: baseline_feature.implemented,
|
||||
functional: baseline_feature.functional,
|
||||
integrated: baseline_feature.integrated,
|
||||
performance_score: self.calculate_performance_score(&baseline_feature.performance_characteristics),
|
||||
};
|
||||
{
|
||||
let baseline_status = FeatureStatus {
|
||||
implemented: baseline_feature.implemented,
|
||||
functional: baseline_feature.functional,
|
||||
integrated: baseline_feature.integrated,
|
||||
performance_score: self
|
||||
.calculate_performance_score(&baseline_feature.performance_characteristics),
|
||||
};
|
||||
|
||||
let current_status = FeatureStatus {
|
||||
implemented: current_feature.implemented,
|
||||
functional: current_feature.functional,
|
||||
integrated: current_feature.integrated,
|
||||
performance_score: self.calculate_performance_score(¤t_feature.performance_characteristics),
|
||||
};
|
||||
let current_status = FeatureStatus {
|
||||
implemented: current_feature.implemented,
|
||||
functional: current_feature.functional,
|
||||
integrated: current_feature.integrated,
|
||||
performance_score: self
|
||||
.calculate_performance_score(¤t_feature.performance_characteristics),
|
||||
};
|
||||
|
||||
// Check for regressions
|
||||
let mut regression_issues = Vec::new();
|
||||
// Check for regressions
|
||||
let mut regression_issues = Vec::new();
|
||||
|
||||
if baseline_status.implemented && !current_status.implemented {
|
||||
regression_issues.push("Feature no longer implemented");
|
||||
}
|
||||
if baseline_status.functional && !current_status.functional {
|
||||
regression_issues.push("Feature no longer functional");
|
||||
}
|
||||
if baseline_status.integrated && !current_status.integrated {
|
||||
regression_issues.push("Feature no longer integrated");
|
||||
}
|
||||
if current_status.performance_score < baseline_status.performance_score * 0.9 {
|
||||
regression_issues.push("Performance score degraded significantly");
|
||||
}
|
||||
if baseline_status.implemented && !current_status.implemented {
|
||||
regression_issues.push("Feature no longer implemented");
|
||||
}
|
||||
if baseline_status.functional && !current_status.functional {
|
||||
regression_issues.push("Feature no longer functional");
|
||||
}
|
||||
if baseline_status.integrated && !current_status.integrated {
|
||||
regression_issues.push("Feature no longer integrated");
|
||||
}
|
||||
if current_status.performance_score < baseline_status.performance_score * 0.9 {
|
||||
regression_issues.push("Performance score degraded significantly");
|
||||
}
|
||||
|
||||
if !regression_issues.is_empty() {
|
||||
regressions.push(FeatureRegression {
|
||||
feature_name: current_feature.feature_name.clone(),
|
||||
regression_type: regression_issues.join(", "),
|
||||
baseline_status,
|
||||
current_status,
|
||||
issue_description: format!(
|
||||
if !regression_issues.is_empty() {
|
||||
regressions.push(FeatureRegression {
|
||||
feature_name: current_feature.feature_name.clone(),
|
||||
regression_type: regression_issues.join(", "),
|
||||
baseline_status,
|
||||
current_status,
|
||||
issue_description: format!(
|
||||
"Feature regression detected: {}",
|
||||
regression_issues.join("; ")
|
||||
),
|
||||
});
|
||||
}
|
||||
),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
regressions
|
||||
}
|
||||
|
||||
/// Calculate performance score from characteristics
|
||||
fn calculate_performance_score(&self, characteristics: &HashMap<String, f64>) -> f64 {
|
||||
if characteristics.is_empty() {
|
||||
return 0.0;
|
||||
}
|
||||
regressions
|
||||
}
|
||||
|
||||
// Simple average of all performance characteristics
|
||||
let sum: f64 = characteristics.values().sum();
|
||||
sum / characteristics.len() as f64
|
||||
/// Calculate performance score from characteristics
|
||||
fn calculate_performance_score(&self, characteristics: &HashMap<String, f64>) -> f64 {
|
||||
if characteristics.is_empty() {
|
||||
return 0.0;
|
||||
}
|
||||
|
||||
/// Detect new issues not present in baseline
|
||||
fn detect_new_issues(
|
||||
// Simple average of all performance characteristics
|
||||
let sum: f64 = characteristics.values().sum();
|
||||
sum / characteristics.len() as f64
|
||||
}
|
||||
|
||||
/// Detect new issues not present in baseline
|
||||
fn detect_new_issues(
|
||||
&self,
|
||||
baseline: &ValidationReport,
|
||||
current: &ValidationReport,
|
||||
) -> Vec<String> {
|
||||
let mut new_issues = Vec::new();
|
||||
) -> Vec<String> {
|
||||
let mut new_issues = Vec::new();
|
||||
|
||||
// Check for new failed tests
|
||||
for current_result in ¤t.performance_results {
|
||||
if !current_result.passed {
|
||||
if let Some(baseline_result) = baseline.performance_results.iter()
|
||||
// Check for new failed tests
|
||||
for current_result in ¤t.performance_results {
|
||||
if !current_result.passed {
|
||||
if let Some(baseline_result) = baseline
|
||||
.performance_results
|
||||
.iter()
|
||||
.find(|r| r.test_name == current_result.test_name)
|
||||
{
|
||||
if baseline_result.passed {
|
||||
new_issues.push(format!(
|
||||
{
|
||||
if baseline_result.passed {
|
||||
new_issues.push(format!(
|
||||
"Performance test '{}' now failing (was passing)",
|
||||
current_result.test_name
|
||||
));
|
||||
}
|
||||
} else {
|
||||
new_issues.push(format!(
|
||||
"New performance test '{}' is failing",
|
||||
));
|
||||
}
|
||||
} else {
|
||||
new_issues.push(format!(
|
||||
"New performance test '{}' is failing",
|
||||
current_result.test_name
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check for new accuracy failures
|
||||
for current_result in ¤t.accuracy_results {
|
||||
if !current_result.passed {
|
||||
if let Some(baseline_result) = baseline
|
||||
.accuracy_results
|
||||
.iter()
|
||||
.find(|r| r.test_name == current_result.test_name)
|
||||
{
|
||||
if baseline_result.passed {
|
||||
new_issues.push(format!(
|
||||
"Accuracy test '{}' now failing (was passing)",
|
||||
current_result.test_name
|
||||
));
|
||||
}
|
||||
));
|
||||
}
|
||||
} else {
|
||||
new_issues.push(format!(
|
||||
"New accuracy test '{}' is failing",
|
||||
current_result.test_name
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check for new feature issues
|
||||
for current_feature in ¤t.feature_results {
|
||||
if !current_feature.issues.is_empty() {
|
||||
if let Some(baseline_feature) = baseline
|
||||
.feature_results
|
||||
.iter()
|
||||
.find(|f| f.feature_name == current_feature.feature_name)
|
||||
{
|
||||
let new_feature_issues: Vec<&String> = current_feature
|
||||
.issues
|
||||
.iter()
|
||||
.filter(|issue| !baseline_feature.issues.contains(issue))
|
||||
.collect();
|
||||
|
||||
for new_issue in new_feature_issues {
|
||||
new_issues.push(format!(
|
||||
"New issue in feature '{}': {}",
|
||||
current_feature.feature_name, new_issue
|
||||
));
|
||||
}
|
||||
} else {
|
||||
// New feature with issues
|
||||
for issue in ¤t_feature.issues {
|
||||
new_issues.push(format!(
|
||||
"New feature '{}' has issue: {}",
|
||||
current_feature.feature_name, issue
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check for new accuracy failures
|
||||
for current_result in ¤t.accuracy_results {
|
||||
if !current_result.passed {
|
||||
if let Some(baseline_result) = baseline.accuracy_results.iter()
|
||||
.find(|r| r.test_name == current_result.test_name)
|
||||
{
|
||||
if baseline_result.passed {
|
||||
new_issues.push(format!(
|
||||
"Accuracy test '{}' now failing (was passing)",
|
||||
current_result.test_name
|
||||
));
|
||||
}
|
||||
} else {
|
||||
new_issues.push(format!(
|
||||
"New accuracy test '{}' is failing",
|
||||
current_result.test_name
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
new_issues
|
||||
}
|
||||
|
||||
// Check for new feature issues
|
||||
for current_feature in ¤t.feature_results {
|
||||
if !current_feature.issues.is_empty() {
|
||||
if let Some(baseline_feature) = baseline.feature_results.iter()
|
||||
.find(|f| f.feature_name == current_feature.feature_name)
|
||||
{
|
||||
let new_feature_issues: Vec<&String> = current_feature.issues.iter()
|
||||
.filter(|issue| !baseline_feature.issues.contains(issue))
|
||||
.collect();
|
||||
/// Generate regression summary and recommendations
|
||||
fn generate_regression_summary(
|
||||
&self,
|
||||
performance_regressions: &[PerformanceRegression],
|
||||
accuracy_regressions: &[AccuracyRegression],
|
||||
feature_regressions: &[FeatureRegression],
|
||||
new_issues: &[String],
|
||||
) -> RegressionSummary {
|
||||
let total_regressions =
|
||||
performance_regressions.len() + accuracy_regressions.len() + feature_regressions.len();
|
||||
|
||||
for new_issue in new_feature_issues {
|
||||
new_issues.push(format!(
|
||||
"New issue in feature '{}': {}",
|
||||
current_feature.feature_name,
|
||||
new_issue
|
||||
));
|
||||
}
|
||||
} else {
|
||||
// New feature with issues
|
||||
for issue in ¤t_feature.issues {
|
||||
new_issues.push(format!(
|
||||
"New feature '{}' has issue: {}",
|
||||
current_feature.feature_name,
|
||||
issue
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
let critical_regressions = performance_regressions
|
||||
.iter()
|
||||
.filter(|r| r.is_critical)
|
||||
.count();
|
||||
|
||||
new_issues
|
||||
}
|
||||
// Determine risk level
|
||||
let risk_level = if critical_regressions > 0 {
|
||||
RiskLevel::Critical
|
||||
} else if total_regressions > 5 || !new_issues.is_empty() {
|
||||
RiskLevel::High
|
||||
} else if total_regressions > 2 {
|
||||
RiskLevel::Medium
|
||||
} else {
|
||||
RiskLevel::Low
|
||||
};
|
||||
|
||||
/// Generate regression summary and recommendations
|
||||
fn generate_regression_summary(
|
||||
&self,
|
||||
performance_regressions: &[PerformanceRegression],
|
||||
accuracy_regressions: &[AccuracyRegression],
|
||||
feature_regressions: &[FeatureRegression],
|
||||
new_issues: &[String],
|
||||
) -> RegressionSummary {
|
||||
let total_regressions = performance_regressions.len() +
|
||||
accuracy_regressions.len() +
|
||||
feature_regressions.len();
|
||||
// Generate recommendations
|
||||
let mut recommended_actions = Vec::new();
|
||||
|
||||
let critical_regressions = performance_regressions.iter()
|
||||
.filter(|r| r.is_critical)
|
||||
.count();
|
||||
if critical_regressions > 0 {
|
||||
recommended_actions.push(
|
||||
"URGENT: Address critical performance regressions before release".to_string(),
|
||||
);
|
||||
}
|
||||
|
||||
// Determine risk level
|
||||
let risk_level = if critical_regressions > 0 {
|
||||
RiskLevel::Critical
|
||||
} else if total_regressions > 5 || !new_issues.is_empty() {
|
||||
RiskLevel::High
|
||||
} else if total_regressions > 2 {
|
||||
RiskLevel::Medium
|
||||
} else {
|
||||
RiskLevel::Low
|
||||
};
|
||||
if !performance_regressions.is_empty() {
|
||||
recommended_actions
|
||||
.push("Review and optimize performance regression areas".to_string());
|
||||
}
|
||||
|
||||
// Generate recommendations
|
||||
let mut recommended_actions = Vec::new();
|
||||
if !accuracy_regressions.is_empty() {
|
||||
recommended_actions.push("Investigate numerical accuracy degradation".to_string());
|
||||
}
|
||||
|
||||
if critical_regressions > 0 {
|
||||
recommended_actions.push("URGENT: Address critical performance regressions before release".to_string());
|
||||
}
|
||||
if !feature_regressions.is_empty() {
|
||||
recommended_actions.push("Fix revolutionary feature regressions".to_string());
|
||||
}
|
||||
|
||||
if !performance_regressions.is_empty() {
|
||||
recommended_actions.push("Review and optimize performance regression areas".to_string());
|
||||
}
|
||||
if !new_issues.is_empty() {
|
||||
recommended_actions.push("Address newly introduced issues".to_string());
|
||||
}
|
||||
|
||||
if !accuracy_regressions.is_empty() {
|
||||
recommended_actions.push("Investigate numerical accuracy degradation".to_string());
|
||||
}
|
||||
if recommended_actions.is_empty() {
|
||||
recommended_actions
|
||||
.push("All regression tests passed - ready for deployment".to_string());
|
||||
}
|
||||
|
||||
if !feature_regressions.is_empty() {
|
||||
recommended_actions.push("Fix revolutionary feature regressions".to_string());
|
||||
}
|
||||
RegressionSummary {
|
||||
total_regressions,
|
||||
critical_regressions,
|
||||
performance_regressions: performance_regressions.len(),
|
||||
accuracy_regressions: accuracy_regressions.len(),
|
||||
feature_regressions: feature_regressions.len(),
|
||||
risk_level,
|
||||
recommended_actions,
|
||||
}
|
||||
}
|
||||
|
||||
if !new_issues.is_empty() {
|
||||
recommended_actions.push("Address newly introduced issues".to_string());
|
||||
}
|
||||
/// Save regression test results
|
||||
pub fn save_results(&self, results: &RegressionTestResult, path: &str) -> Result<()> {
|
||||
let json = serde_json::to_string_pretty(results)
|
||||
.map_err(|e| TransformerError::SerializationError(e.to_string()))?;
|
||||
|
||||
if recommended_actions.is_empty() {
|
||||
recommended_actions.push("All regression tests passed - ready for deployment".to_string());
|
||||
}
|
||||
std::fs::write(path, json).map_err(TransformerError::IoError)?;
|
||||
|
||||
RegressionSummary {
|
||||
total_regressions,
|
||||
critical_regressions,
|
||||
performance_regressions: performance_regressions.len(),
|
||||
accuracy_regressions: accuracy_regressions.len(),
|
||||
feature_regressions: feature_regressions.len(),
|
||||
risk_level,
|
||||
recommended_actions,
|
||||
}
|
||||
}
|
||||
tracing::info!("Regression test results saved to: {}", path);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Save regression test results
|
||||
pub fn save_results(&self, results: &RegressionTestResult, path: &str) -> Result<()> {
|
||||
let json = serde_json::to_string_pretty(results)
|
||||
.map_err(|e| TransformerError::SerializationError(e.to_string()))?;
|
||||
/// Update baseline report with current validation
|
||||
pub fn update_baseline(&mut self) -> Result<()> {
|
||||
tracing::info!("Updating baseline validation report");
|
||||
|
||||
std::fs::write(path, json)
|
||||
.map_err(TransformerError::IoError)?;
|
||||
let current_report = self.validation_framework.run_comprehensive_validation()?;
|
||||
self.validation_framework
|
||||
.save_report(¤t_report, &self.config.baseline_report_path)?;
|
||||
|
||||
tracing::info!("Regression test results saved to: {}", path);
|
||||
Ok(())
|
||||
}
|
||||
tracing::info!("Baseline report updated successfully");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Update baseline report with current validation
|
||||
pub fn update_baseline(&mut self) -> Result<()> {
|
||||
tracing::info!("Updating baseline validation report");
|
||||
/// Run continuous regression monitoring
|
||||
pub fn run_continuous_monitoring(&mut self, interval_seconds: u64) -> Result<()> {
|
||||
tracing::info!(
|
||||
"Starting continuous regression monitoring (interval: {}s)",
|
||||
interval_seconds
|
||||
);
|
||||
|
||||
let current_report = self.validation_framework.run_comprehensive_validation()?;
|
||||
self.validation_framework.save_report(¤t_report, &self.config.baseline_report_path)?;
|
||||
loop {
|
||||
let results = self.run_regression_tests()?;
|
||||
|
||||
tracing::info!("Baseline report updated successfully");
|
||||
Ok(())
|
||||
}
|
||||
if results.summary.risk_level == RiskLevel::Critical {
|
||||
tracing::error!("CRITICAL REGRESSION DETECTED!");
|
||||
for action in &results.summary.recommended_actions {
|
||||
tracing::error!("ACTION REQUIRED: {}", action);
|
||||
}
|
||||
|
||||
/// Run continuous regression monitoring
|
||||
pub fn run_continuous_monitoring(&mut self, interval_seconds: u64) -> Result<()> {
|
||||
tracing::info!("Starting continuous regression monitoring (interval: {}s)", interval_seconds);
|
||||
// Save critical results
|
||||
let timestamp = chrono::Utc::now().format("%Y%m%d_%H%M%S");
|
||||
let critical_path = format!("critical_regression_{}.json", timestamp);
|
||||
self.save_results(&results, &critical_path)?;
|
||||
|
||||
loop {
|
||||
let results = self.run_regression_tests()?;
|
||||
return Err(TransformerError::ComputationError(
|
||||
"Critical regression detected - monitoring stopped".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
if results.summary.risk_level == RiskLevel::Critical {
|
||||
tracing::error!("CRITICAL REGRESSION DETECTED!");
|
||||
for action in &results.summary.recommended_actions {
|
||||
tracing::error!("ACTION REQUIRED: {}", action);
|
||||
}
|
||||
if results.summary.risk_level == RiskLevel::High {
|
||||
tracing::warn!("High risk regression detected:");
|
||||
for action in &results.summary.recommended_actions {
|
||||
tracing::warn!(" {}", action);
|
||||
}
|
||||
}
|
||||
|
||||
// Save critical results
|
||||
let timestamp = chrono::Utc::now().format("%Y%m%d_%H%M%S");
|
||||
let critical_path = format!("critical_regression_{}.json", timestamp);
|
||||
self.save_results(&results, &critical_path)?;
|
||||
tracing::info!(
|
||||
"Regression check complete: {} total regressions, risk level: {:?}",
|
||||
results.summary.total_regressions,
|
||||
results.summary.risk_level
|
||||
);
|
||||
|
||||
return Err(TransformerError::ComputationError(
|
||||
"Critical regression detected - monitoring stopped".to_string()
|
||||
));
|
||||
}
|
||||
// Wait for next check
|
||||
std::thread::sleep(std::time::Duration::from_secs(interval_seconds));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if results.summary.risk_level == RiskLevel::High {
|
||||
tracing::warn!("High risk regression detected:");
|
||||
for action in &results.summary.recommended_actions {
|
||||
tracing::warn!(" {}", action);
|
||||
}
|
||||
}
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
tracing::info!(
|
||||
"Regression check complete: {} total regressions, risk level: {:?}",
|
||||
results.summary.total_regressions,
|
||||
results.summary.risk_level
|
||||
);
|
||||
#[test]
|
||||
fn test_regression_test_suite_creation() {
|
||||
let suite = RegressionTestSuite::default();
|
||||
assert_eq!(suite.config.max_performance_regression, 0.05);
|
||||
assert_eq!(suite.config.max_accuracy_degradation, 0.01);
|
||||
}
|
||||
|
||||
// Wait for next check
|
||||
std::thread::sleep(std::time::Duration::from_secs(interval_seconds));
|
||||
}
|
||||
}
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn test_regression_detection() {
|
||||
let mut suite = RegressionTestSuite::new(RegressionTestConfig {
|
||||
baseline_report_path: "/tmp/test_baseline.json".to_string(),
|
||||
..RegressionTestConfig::default()
|
||||
});
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
// This should create a baseline and then test against it
|
||||
let results = suite.run_regression_tests().unwrap();
|
||||
|
||||
#[test]
|
||||
fn test_regression_test_suite_creation() {
|
||||
let suite = RegressionTestSuite::default();
|
||||
assert_eq!(suite.config.max_performance_regression, 0.05);
|
||||
assert_eq!(suite.config.max_accuracy_degradation, 0.01);
|
||||
}
|
||||
// First run should have no regressions (comparing against itself)
|
||||
assert_eq!(results.summary.total_regressions, 0);
|
||||
assert_eq!(results.summary.risk_level, RiskLevel::Low);
|
||||
assert!(results.all_tests_passed);
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_regression_detection() {
|
||||
let mut suite = RegressionTestSuite::new(RegressionTestConfig {
|
||||
baseline_report_path: "/tmp/test_baseline.json".to_string(),
|
||||
..RegressionTestConfig::default()
|
||||
});
|
||||
// Clean up
|
||||
std::fs::remove_file("/tmp/test_baseline.json").ok();
|
||||
}
|
||||
|
||||
// This should create a baseline and then test against it
|
||||
let results = suite.run_regression_tests().unwrap();
|
||||
#[test]
|
||||
fn test_performance_regression_analysis() {
|
||||
let config = RegressionTestConfig::default();
|
||||
let suite = RegressionTestSuite::new(config);
|
||||
|
||||
// First run should have no regressions (comparing against itself)
|
||||
assert_eq!(results.summary.total_regressions, 0);
|
||||
assert_eq!(results.summary.risk_level, RiskLevel::Low);
|
||||
assert!(results.all_tests_passed);
|
||||
// Create mock reports
|
||||
let baseline = create_mock_baseline_report();
|
||||
let current = create_mock_current_report_with_regression();
|
||||
|
||||
// Clean up
|
||||
std::fs::remove_file("/tmp/test_baseline.json").ok();
|
||||
}
|
||||
let regressions = suite.analyze_performance_regressions(&baseline, ¤t);
|
||||
|
||||
#[test]
|
||||
fn test_performance_regression_analysis() {
|
||||
let config = RegressionTestConfig::default();
|
||||
let suite = RegressionTestSuite::new(config);
|
||||
assert!(!regressions.is_empty());
|
||||
assert!(regressions[0].regression_percentage < 0.0); // Should detect regression
|
||||
}
|
||||
|
||||
// Create mock reports
|
||||
let baseline = create_mock_baseline_report();
|
||||
let current = create_mock_current_report_with_regression();
|
||||
#[test]
|
||||
fn test_risk_level_calculation() {
|
||||
let suite = RegressionTestSuite::default();
|
||||
|
||||
let regressions = suite.analyze_performance_regressions(&baseline, ¤t);
|
||||
// Test critical risk (with critical regressions)
|
||||
let perf_regressions = vec![PerformanceRegression {
|
||||
test_name: "Critical Test".to_string(),
|
||||
baseline_performance: 10.0,
|
||||
current_performance: 5.0,
|
||||
regression_percentage: -50.0,
|
||||
is_critical: true,
|
||||
context: HashMap::new(),
|
||||
}];
|
||||
|
||||
assert!(!regressions.is_empty());
|
||||
assert!(regressions[0].regression_percentage < 0.0); // Should detect regression
|
||||
}
|
||||
let summary = suite.generate_regression_summary(&perf_regressions, &[], &[], &[]);
|
||||
assert_eq!(summary.risk_level, RiskLevel::Critical);
|
||||
assert!(summary.recommended_actions[0].contains("URGENT"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_risk_level_calculation() {
|
||||
let suite = RegressionTestSuite::default();
|
||||
fn create_mock_baseline_report() -> ValidationReport {
|
||||
ValidationReport {
|
||||
timestamp: chrono::Utc::now(),
|
||||
performance_results: vec![PerformanceValidationResult {
|
||||
test_name: "Flash Attention".to_string(),
|
||||
expected_performance: 6.0,
|
||||
actual_performance: 6.5, // Good baseline
|
||||
passed: true,
|
||||
threshold: 0.8,
|
||||
metadata: HashMap::new(),
|
||||
}],
|
||||
accuracy_results: vec![],
|
||||
feature_results: vec![],
|
||||
overall_pass_rate: 0.9,
|
||||
summary: ValidationSummary {
|
||||
total_tests: 1,
|
||||
tests_passed: 1,
|
||||
tests_failed: 0,
|
||||
revolutionary_features_count: 4,
|
||||
performance_claims_validated: 1,
|
||||
key_achievements: vec![],
|
||||
improvement_areas: vec![],
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Test critical risk (with critical regressions)
|
||||
let perf_regressions = vec![PerformanceRegression {
|
||||
test_name: "Critical Test".to_string(),
|
||||
baseline_performance: 10.0,
|
||||
current_performance: 5.0,
|
||||
regression_percentage: -50.0,
|
||||
is_critical: true,
|
||||
context: HashMap::new(),
|
||||
}];
|
||||
|
||||
let summary = suite.generate_regression_summary(&perf_regressions, &[], &[], &[]);
|
||||
assert_eq!(summary.risk_level, RiskLevel::Critical);
|
||||
assert!(summary.recommended_actions[0].contains("URGENT"));
|
||||
}
|
||||
|
||||
fn create_mock_baseline_report() -> ValidationReport {
|
||||
ValidationReport {
|
||||
timestamp: chrono::Utc::now(),
|
||||
performance_results: vec![
|
||||
PerformanceValidationResult {
|
||||
test_name: "Flash Attention".to_string(),
|
||||
expected_performance: 6.0,
|
||||
actual_performance: 6.5, // Good baseline
|
||||
passed: true,
|
||||
threshold: 0.8,
|
||||
metadata: HashMap::new(),
|
||||
}
|
||||
],
|
||||
accuracy_results: vec![],
|
||||
feature_results: vec![],
|
||||
overall_pass_rate: 0.9,
|
||||
summary: ValidationSummary {
|
||||
total_tests: 1,
|
||||
tests_passed: 1,
|
||||
tests_failed: 0,
|
||||
revolutionary_features_count: 4,
|
||||
performance_claims_validated: 1,
|
||||
key_achievements: vec![],
|
||||
improvement_areas: vec![],
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn create_mock_current_report_with_regression() -> ValidationReport {
|
||||
ValidationReport {
|
||||
timestamp: chrono::Utc::now(),
|
||||
performance_results: vec![
|
||||
PerformanceValidationResult {
|
||||
test_name: "Flash Attention".to_string(),
|
||||
expected_performance: 6.0,
|
||||
actual_performance: 5.0, // Regressed from 6.5 to 5.0
|
||||
passed: false,
|
||||
threshold: 0.8,
|
||||
metadata: HashMap::new(),
|
||||
}
|
||||
],
|
||||
accuracy_results: vec![],
|
||||
feature_results: vec![],
|
||||
overall_pass_rate: 0.7, // Worse than baseline
|
||||
summary: ValidationSummary {
|
||||
total_tests: 1,
|
||||
tests_passed: 0,
|
||||
tests_failed: 1,
|
||||
revolutionary_features_count: 4,
|
||||
performance_claims_validated: 0,
|
||||
key_achievements: vec![],
|
||||
improvement_areas: vec!["Performance regression".to_string()],
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
fn create_mock_current_report_with_regression() -> ValidationReport {
|
||||
ValidationReport {
|
||||
timestamp: chrono::Utc::now(),
|
||||
performance_results: vec![PerformanceValidationResult {
|
||||
test_name: "Flash Attention".to_string(),
|
||||
expected_performance: 6.0,
|
||||
actual_performance: 5.0, // Regressed from 6.5 to 5.0
|
||||
passed: false,
|
||||
threshold: 0.8,
|
||||
metadata: HashMap::new(),
|
||||
}],
|
||||
accuracy_results: vec![],
|
||||
feature_results: vec![],
|
||||
overall_pass_rate: 0.7, // Worse than baseline
|
||||
summary: ValidationSummary {
|
||||
total_tests: 1,
|
||||
tests_passed: 0,
|
||||
tests_failed: 1,
|
||||
revolutionary_features_count: 4,
|
||||
performance_claims_validated: 0,
|
||||
key_achievements: vec![],
|
||||
improvement_areas: vec!["Performance regression".to_string()],
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user