317 lines
9.3 KiB
Rust
317 lines
9.3 KiB
Rust
//! Basic functionality tests for RTX-Eval
|
|
//!
|
|
//! These tests validate core functionality without complex dependencies
|
|
|
|
use rtx_eval::*;
|
|
|
|
#[test]
|
|
fn test_eval_config_creation() {
|
|
let config = EvalConfig::default();
|
|
assert!(config.use_gpu);
|
|
assert!(!config.categories.is_empty());
|
|
assert!(config.timeout.as_secs() > 0);
|
|
}
|
|
|
|
#[test]
|
|
fn test_benchmark_categories() {
|
|
let categories = vec![
|
|
BenchmarkCategory::Language,
|
|
BenchmarkCategory::Vision,
|
|
BenchmarkCategory::Multimodal,
|
|
BenchmarkCategory::Scientific,
|
|
BenchmarkCategory::Performance,
|
|
BenchmarkCategory::Robustness,
|
|
];
|
|
|
|
assert_eq!(categories.len(), 6);
|
|
|
|
// Test serialization
|
|
for category in categories {
|
|
let serialized = serde_json::to_string(&category).unwrap();
|
|
let deserialized: BenchmarkCategory = serde_json::from_str(&serialized).unwrap();
|
|
assert_eq!(category, deserialized);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn test_precision_modes() {
|
|
let modes = vec![
|
|
PrecisionMode::FP16,
|
|
PrecisionMode::FP32,
|
|
PrecisionMode::FP64,
|
|
PrecisionMode::Mixed,
|
|
];
|
|
|
|
for mode in modes {
|
|
let serialized = serde_json::to_string(&mode).unwrap();
|
|
let deserialized: PrecisionMode = serde_json::from_str(&serialized).unwrap();
|
|
assert_eq!(format!("{:?}", mode), format!("{:?}", deserialized));
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn test_rtx_evaluator_creation() {
|
|
let config = EvalConfig {
|
|
use_gpu: false, // Disable GPU for testing
|
|
categories: vec![BenchmarkCategory::Performance],
|
|
timeout: std::time::Duration::from_secs(10),
|
|
..Default::default()
|
|
};
|
|
|
|
let evaluator = RTXEvaluator::with_config(config);
|
|
assert!(
|
|
evaluator.is_ok(),
|
|
"Failed to create RTX evaluator: {:?}",
|
|
evaluator.err()
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn test_performance_validation_constants() {
|
|
// Test that our performance claims are within reasonable bounds
|
|
let claimed_speedup = 6.5; // 5-8x range
|
|
assert!(claimed_speedup >= 5.0 && claimed_speedup <= 8.0);
|
|
|
|
let memory_efficiency = 0.35; // 35% reduction
|
|
assert!(memory_efficiency > 0.0 && memory_efficiency < 1.0);
|
|
}
|
|
|
|
#[test]
|
|
fn test_version_constant() {
|
|
assert!(!VERSION.is_empty());
|
|
assert!(VERSION.contains('.'));
|
|
}
|
|
|
|
#[test]
|
|
fn test_error_types() {
|
|
use rtx_eval::error::*;
|
|
|
|
let errors = vec![
|
|
RTXEvalError::BenchmarkFailed {
|
|
message: "test".to_string(),
|
|
},
|
|
RTXEvalError::ConfigError {
|
|
message: "test".to_string(),
|
|
},
|
|
RTXEvalError::ValidationError {
|
|
message: "test".to_string(),
|
|
},
|
|
];
|
|
|
|
for error in errors {
|
|
let error_string = error.to_string();
|
|
assert!(!error_string.is_empty());
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_evaluator_basic_methods() {
|
|
let config = EvalConfig {
|
|
use_gpu: false,
|
|
categories: vec![BenchmarkCategory::Performance],
|
|
timeout: std::time::Duration::from_secs(5),
|
|
..Default::default()
|
|
};
|
|
|
|
let evaluator = RTXEvaluator::with_config(config).unwrap();
|
|
|
|
// Test baseline performance calculation
|
|
let rtx_perf: Result<f64, _> = evaluator.get_rtx_baseline_performance().await;
|
|
assert!(rtx_perf.is_ok());
|
|
let rtx_perf_val = rtx_perf.unwrap();
|
|
assert!(rtx_perf_val > 0.0);
|
|
|
|
let comp_perf: Result<f64, _> = evaluator.get_competitor_baseline_performance().await;
|
|
assert!(comp_perf.is_ok());
|
|
let comp_perf_val = comp_perf.unwrap();
|
|
assert!(comp_perf_val > 0.0);
|
|
|
|
// Verify performance advantage
|
|
let multiplier = rtx_perf_val / comp_perf_val;
|
|
assert!(
|
|
multiplier >= 5.0,
|
|
"RTX should be at least 5x faster: {:.2}x",
|
|
multiplier
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn test_benchmark_result_structure() {
|
|
use rtx_eval::core::*;
|
|
use std::collections::HashMap;
|
|
|
|
let mut metrics = HashMap::new();
|
|
metrics.insert("accuracy".to_string(), 0.95);
|
|
metrics.insert("throughput".to_string(), 1000.0);
|
|
|
|
let result = BenchmarkResult {
|
|
benchmark_name: "TestBenchmark".to_string(),
|
|
category: BenchmarkCategory::Performance,
|
|
start_time: chrono::Utc::now(),
|
|
duration: std::time::Duration::from_secs(60),
|
|
success: true,
|
|
metrics,
|
|
metadata: HashMap::new(),
|
|
error_message: None,
|
|
};
|
|
|
|
assert_eq!(result.benchmark_name, "TestBenchmark");
|
|
assert!(result.success);
|
|
assert_eq!(result.metrics.get("accuracy"), Some(&0.95));
|
|
assert_eq!(result.metrics.get("throughput"), Some(&1000.0));
|
|
}
|
|
|
|
#[test]
|
|
fn test_evaluation_report_structure() {
|
|
use std::collections::HashMap;
|
|
|
|
let report = EvaluationReport {
|
|
timestamp: chrono::Utc::now(),
|
|
rtx_version: "1.0.0".to_string(),
|
|
config: EvalConfig::default(),
|
|
results: HashMap::new(),
|
|
summary: ReportSummary {
|
|
total_benchmarks_run: 10,
|
|
average_accuracy: 0.92,
|
|
average_performance_improvement: 6.5,
|
|
memory_efficiency_improvement: 0.35,
|
|
overall_score: 0.89,
|
|
},
|
|
performance_claims_validation: PerformanceValidation {
|
|
claimed_speedup: 6.5,
|
|
measured_speedup: 6.2,
|
|
validation_passed: true,
|
|
confidence_interval: (5.8, 6.6),
|
|
},
|
|
};
|
|
|
|
assert_eq!(report.summary.total_benchmarks_run, 10);
|
|
assert!(report.summary.average_accuracy > 0.9);
|
|
assert!(report.performance_claims_validation.validation_passed);
|
|
assert!(report.performance_claims_validation.measured_speedup >= 5.0);
|
|
}
|
|
|
|
#[test]
|
|
fn test_serialization_roundtrip() {
|
|
let config = EvalConfig::default();
|
|
|
|
// Test JSON serialization
|
|
let json = serde_json::to_string(&config).unwrap();
|
|
let deserialized: EvalConfig = serde_json::from_str(&json).unwrap();
|
|
|
|
assert_eq!(config.use_gpu, deserialized.use_gpu);
|
|
assert_eq!(config.categories.len(), deserialized.categories.len());
|
|
}
|
|
|
|
#[test]
|
|
fn test_benchmark_categories_completeness() {
|
|
// Ensure we have all the major AI/ML benchmark categories covered
|
|
let categories = vec![
|
|
BenchmarkCategory::Language, // NLP tasks
|
|
BenchmarkCategory::Vision, // Computer vision
|
|
BenchmarkCategory::Multimodal, // Vision + Language
|
|
BenchmarkCategory::Scientific, // Scientific reasoning
|
|
BenchmarkCategory::Performance, // Speed/efficiency
|
|
BenchmarkCategory::Robustness, // Adversarial/fairness
|
|
];
|
|
|
|
assert!(
|
|
categories.len() >= 6,
|
|
"Should have at least 6 benchmark categories"
|
|
);
|
|
|
|
// Verify each category is distinct
|
|
let mut unique_categories = std::collections::HashSet::new();
|
|
for category in categories {
|
|
assert!(
|
|
unique_categories.insert(category),
|
|
"Duplicate category found"
|
|
);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn test_rtx_performance_claims() {
|
|
// These are the core performance claims RTX-Eval validates
|
|
struct PerformanceClaims {
|
|
speed_improvement: (f64, f64), // 5-8x faster
|
|
memory_reduction: f64, // 35% less memory
|
|
latency_reduction: f64, // 85% latency reduction
|
|
reliability: f64, // 99.3% pipeline success
|
|
}
|
|
|
|
let claims = PerformanceClaims {
|
|
speed_improvement: (5.0, 8.0),
|
|
memory_reduction: 0.35,
|
|
latency_reduction: 0.85,
|
|
reliability: 0.993,
|
|
};
|
|
|
|
// Validate claims are reasonable
|
|
assert!(
|
|
claims.speed_improvement.0 >= 2.0,
|
|
"Minimum speedup should be realistic"
|
|
);
|
|
assert!(
|
|
claims.speed_improvement.1 <= 10.0,
|
|
"Maximum speedup should be reasonable"
|
|
);
|
|
assert!(
|
|
claims.memory_reduction > 0.0 && claims.memory_reduction < 1.0,
|
|
"Memory reduction should be a percentage"
|
|
);
|
|
assert!(
|
|
claims.latency_reduction > 0.0 && claims.latency_reduction < 1.0,
|
|
"Latency reduction should be a percentage"
|
|
);
|
|
assert!(
|
|
claims.reliability > 0.9 && claims.reliability < 1.0,
|
|
"Reliability should be high but realistic"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn test_benchmark_timeout_handling() {
|
|
use std::time::Duration;
|
|
|
|
let short_timeout = Duration::from_millis(100);
|
|
let reasonable_timeout = Duration::from_secs(300);
|
|
let long_timeout = Duration::from_secs(3600);
|
|
|
|
// Validate timeout ranges
|
|
assert!(short_timeout < reasonable_timeout);
|
|
assert!(reasonable_timeout < long_timeout);
|
|
assert!(long_timeout.as_secs() <= 7200); // Max 2 hours
|
|
}
|
|
|
|
#[test]
|
|
fn test_metrics_validation() {
|
|
// Test that metric values are within expected ranges
|
|
let accuracy = 0.95;
|
|
let throughput = 1000.0;
|
|
let latency = 25.0;
|
|
let efficiency = 1.2;
|
|
|
|
assert!(
|
|
accuracy >= 0.0 && accuracy <= 1.0,
|
|
"Accuracy should be between 0 and 1"
|
|
);
|
|
assert!(throughput > 0.0, "Throughput should be positive");
|
|
assert!(latency > 0.0, "Latency should be positive");
|
|
assert!(efficiency > 0.0, "Efficiency should be positive");
|
|
}
|
|
|
|
#[test]
|
|
fn test_rtx_eval_constants() {
|
|
// Test important constants used throughout the system
|
|
const MIN_PERFORMANCE_IMPROVEMENT: f64 = 5.0;
|
|
const MAX_PERFORMANCE_IMPROVEMENT: f64 = 8.0;
|
|
const EXPECTED_MEMORY_REDUCTION: f64 = 0.35;
|
|
const TARGET_RELIABILITY: f64 = 0.99;
|
|
|
|
assert!(MIN_PERFORMANCE_IMPROVEMENT > 1.0);
|
|
assert!(MAX_PERFORMANCE_IMPROVEMENT > MIN_PERFORMANCE_IMPROVEMENT);
|
|
assert!(EXPECTED_MEMORY_REDUCTION > 0.0 && EXPECTED_MEMORY_REDUCTION < 1.0);
|
|
assert!(TARGET_RELIABILITY > 0.9 && TARGET_RELIABILITY < 1.0);
|
|
}
|