270 lines
8.4 KiB
Rust
270 lines
8.4 KiB
Rust
//! Integration tests for RTX benchmark suite
|
|
#![cfg(feature = "disabled_tests")]
|
|
|
|
use anyhow::Result;
|
|
use rtx_bench::{
|
|
BenchmarkConfig, BenchmarkSuite,
|
|
core::CoreInfrastructureBenchmarks,
|
|
models::ModelArchitectureBenchmarks,
|
|
reports::{ReportConfig, ReportGenerator},
|
|
utils::SystemInfoCollector,
|
|
};
|
|
use tempfile::TempDir;
|
|
|
|
#[tokio::test]
|
|
async fn test_benchmark_suite_creation() {
|
|
let config = BenchmarkConfig::default();
|
|
let suite = BenchmarkSuite::new(config);
|
|
// Test passes if construction succeeds
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_core_infrastructure_benchmarks() -> Result<()> {
|
|
let benchmarks = CoreInfrastructureBenchmarks::new();
|
|
let config = BenchmarkConfig::default()
|
|
.with_measurement_iterations(2)
|
|
.with_warmup_iterations(1);
|
|
|
|
let results = benchmarks.run_all(&config).await?;
|
|
|
|
assert_eq!(results.suite_name, "CoreInfrastructure");
|
|
assert!(!results.benchmarks.is_empty());
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_model_architecture_benchmarks() -> Result<()> {
|
|
let benchmarks = ModelArchitectureBenchmarks::new();
|
|
let config = BenchmarkConfig::default()
|
|
.with_measurement_iterations(2)
|
|
.with_warmup_iterations(1);
|
|
|
|
let results = benchmarks.run_all(&config).await?;
|
|
|
|
assert_eq!(results.suite_name, "ModelArchitectures");
|
|
assert!(!results.benchmarks.is_empty());
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_report_generation() -> Result<()> {
|
|
let config = BenchmarkConfig::default()
|
|
.with_measurement_iterations(2)
|
|
.with_warmup_iterations(1);
|
|
|
|
// Generate some test results
|
|
let core_benchmarks = CoreInfrastructureBenchmarks::new();
|
|
let results = vec![core_benchmarks.run_all(&config).await?];
|
|
|
|
// Generate report
|
|
let report_config = ReportConfig::default();
|
|
let generator = ReportGenerator::new(report_config.clone());
|
|
let report = generator.generate_report(&results, &report_config).await?;
|
|
|
|
assert_eq!(
|
|
report.metadata.total_benchmarks,
|
|
results[0].benchmarks.len()
|
|
);
|
|
assert!(!report.benchmarks.is_empty());
|
|
|
|
// Test report export
|
|
let temp_dir = TempDir::new()?;
|
|
generator
|
|
.export_report(&report, temp_dir.path(), &report_config)
|
|
.await?;
|
|
|
|
// Check that files were created
|
|
let json_file = temp_dir.path().join("benchmark_report.json");
|
|
let html_file = temp_dir.path().join("benchmark_report.html");
|
|
|
|
assert!(json_file.exists());
|
|
assert!(html_file.exists());
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[test]
|
|
fn test_system_info_collection() -> Result<()> {
|
|
let system_info = SystemInfoCollector::collect()?;
|
|
|
|
assert!(!system_info.hostname.is_empty());
|
|
assert!(!system_info.os.is_empty());
|
|
assert!(system_info.cpu.cores > 0);
|
|
assert!(system_info.memory.total_mb > 0);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_benchmark_suite_full_run() -> Result<()> {
|
|
let config = BenchmarkConfig::default()
|
|
.with_measurement_iterations(1)
|
|
.with_warmup_iterations(1)
|
|
.with_distributed_enabled(false); // Disable distributed for testing
|
|
|
|
let suite = BenchmarkSuite::new(config);
|
|
let results = suite.run_all_benchmarks().await?;
|
|
|
|
// Should have at least core and model results
|
|
assert!(results.len() >= 2);
|
|
|
|
for result in &results {
|
|
assert!(!result.benchmarks.is_empty());
|
|
assert!(result.total_duration().as_millis() > 0);
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_benchmark_result_serialization() -> Result<()> {
|
|
let config = BenchmarkConfig::default()
|
|
.with_measurement_iterations(2)
|
|
.with_warmup_iterations(1);
|
|
|
|
let benchmarks = CoreInfrastructureBenchmarks::new();
|
|
let results = benchmarks.run_all(&config).await?;
|
|
|
|
// Test JSON serialization
|
|
let json = serde_json::to_string(&results)?;
|
|
assert!(!json.is_empty());
|
|
|
|
// Test deserialization
|
|
let deserialized: rtx_bench::BenchmarkResults = serde_json::from_str(&json)?;
|
|
assert_eq!(deserialized.suite_name, results.suite_name);
|
|
assert_eq!(deserialized.benchmarks.len(), results.benchmarks.len());
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[test]
|
|
fn test_benchmark_config_validation() {
|
|
let config = BenchmarkConfig::default()
|
|
.with_warmup_iterations(0)
|
|
.with_measurement_iterations(1)
|
|
.with_confidence_level(0.95);
|
|
|
|
assert_eq!(config.warmup_iterations, 0);
|
|
assert_eq!(config.measurement_iterations, 1);
|
|
assert_eq!(config.confidence_level, 0.95);
|
|
|
|
// Test confidence level clamping
|
|
let config = BenchmarkConfig::default().with_confidence_level(1.5);
|
|
assert_eq!(config.confidence_level, 1.0);
|
|
|
|
let config = BenchmarkConfig::default().with_confidence_level(-0.5);
|
|
assert_eq!(config.confidence_level, 0.0);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_individual_benchmark_components() -> Result<()> {
|
|
let config = BenchmarkConfig::default()
|
|
.with_measurement_iterations(3)
|
|
.with_warmup_iterations(1);
|
|
|
|
// Test tensor benchmarks individually
|
|
let tensor_benchmarks = rtx_bench::core::tensor_ops::TensorOperationBenchmarks::new();
|
|
let tensor_results: std::collections::HashMap<String, Vec<rtx_bench::BenchmarkMeasurement>> =
|
|
tensor_benchmarks.run_creation_benchmarks(&config).await?;
|
|
assert!(!tensor_results.is_empty());
|
|
|
|
// Test memory benchmarks individually
|
|
let memory_benchmarks = rtx_bench::core::memory::MemoryBenchmarks::new();
|
|
let memory_results: std::collections::HashMap<String, Vec<rtx_bench::BenchmarkMeasurement>> =
|
|
memory_benchmarks.run_allocation_benchmarks(&config).await?;
|
|
assert!(!memory_results.is_empty());
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_benchmark_error_handling() {
|
|
// Test with invalid configuration (this should not panic)
|
|
let config = BenchmarkConfig::default().with_measurement_iterations(0); // Invalid: zero iterations
|
|
|
|
let benchmarks = CoreInfrastructureBenchmarks::new();
|
|
|
|
// This should handle the error gracefully
|
|
let result = benchmarks.run_all(&config).await;
|
|
// We expect this to either succeed with empty results or fail gracefully
|
|
match result {
|
|
Ok(results) => {
|
|
// If it succeeds, results should be valid
|
|
assert!(results.benchmarks.is_empty() || !results.benchmarks.is_empty());
|
|
}
|
|
Err(_) => {
|
|
// If it fails, that's acceptable for invalid configuration
|
|
}
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_competitive_benchmarks() -> Result<()> {
|
|
let config = BenchmarkConfig::default()
|
|
.with_measurement_iterations(2)
|
|
.with_warmup_iterations(1);
|
|
|
|
let benchmarks = rtx_bench::competitive::CompetitiveAnalysisBenchmarks::new();
|
|
let results = benchmarks.run_all(&config).await?;
|
|
|
|
assert_eq!(results.suite_name, "CompetitiveAnalysis");
|
|
// Competitive benchmarks should have measurements with custom metrics
|
|
for (_, stats) in &results.benchmarks {
|
|
// Some benchmarks should have custom metrics for comparisons
|
|
// This is tested in the individual benchmark implementations
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[test]
|
|
fn test_statistical_calculations() -> Result<()> {
|
|
use rtx_bench::utils::StatisticsCalculator;
|
|
|
|
let data = vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0];
|
|
|
|
let p50 = StatisticsCalculator::percentile(&data, 50.0);
|
|
assert_eq!(p50, 5.5); // Median of 1-10
|
|
|
|
let p95 = StatisticsCalculator::percentile(&data, 95.0);
|
|
assert_eq!(p95, 10.0);
|
|
|
|
let confidence_interval = StatisticsCalculator::confidence_interval(&data, 0.95)?;
|
|
assert!(confidence_interval.0 < confidence_interval.1);
|
|
|
|
let t_stat = StatisticsCalculator::welch_t_test(&data[0..5], &data[5..10])?;
|
|
assert!(t_stat > 0.0);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[test]
|
|
fn test_data_formatting() {
|
|
use rtx_bench::utils::DataFormatter;
|
|
use std::time::Duration;
|
|
|
|
// Test duration formatting
|
|
assert_eq!(
|
|
DataFormatter::format_duration(Duration::from_nanos(500)),
|
|
"500ns"
|
|
);
|
|
assert_eq!(
|
|
DataFormatter::format_duration(Duration::from_micros(1500)),
|
|
"1.5μs"
|
|
);
|
|
assert_eq!(
|
|
DataFormatter::format_duration(Duration::from_millis(2500)),
|
|
"2.500s"
|
|
);
|
|
|
|
// Test byte formatting
|
|
assert_eq!(DataFormatter::format_bytes(1024), "1.0 KB");
|
|
assert_eq!(DataFormatter::format_bytes(1536), "1.5 KB");
|
|
assert_eq!(DataFormatter::format_bytes(1048576), "1.0 MB");
|
|
|
|
// Test percentage formatting
|
|
assert_eq!(DataFormatter::format_percentage(0.001), "0.001%");
|
|
assert_eq!(DataFormatter::format_percentage(0.1), "0.10%");
|
|
assert_eq!(DataFormatter::format_percentage(15.5), "15.5%");
|
|
}
|