619 lines
20 KiB
Rust
619 lines
20 KiB
Rust
//! RTX-Eval Benchmark Runner Example
|
|
//!
|
|
//! This example demonstrates how to use RTX-Eval to run comprehensive AI/ML benchmarks
|
|
//! and validate RTX's performance claims across different domains.
|
|
//!
|
|
//! Usage:
|
|
//! ```bash
|
|
//! cargo run --example benchmark_runner -- --help
|
|
//! cargo run --example benchmark_runner -- --category all --output results.json
|
|
//! cargo run --example benchmark_runner -- --category language --compare-competitors
|
|
//! ```
|
|
|
|
use anyhow::Result;
|
|
use clap::{Arg, Command};
|
|
use rtx_eval::automation::*;
|
|
use rtx_eval::validation::*;
|
|
use rtx_eval::*;
|
|
use std::fs;
|
|
use std::path::PathBuf;
|
|
use std::time::Duration;
|
|
use tokio;
|
|
use tracing::{Level, info, warn};
|
|
use tracing_subscriber;
|
|
|
|
#[tokio::main]
|
|
async fn main() -> Result<()> {
|
|
// Initialize logging
|
|
tracing_subscriber::fmt()
|
|
.with_max_level(Level::INFO)
|
|
.with_target(false)
|
|
.init();
|
|
|
|
let matches = Command::new("RTX-Eval Benchmark Runner")
|
|
.version("1.0.0")
|
|
.about("Comprehensive AI/ML benchmarking suite for RTX")
|
|
.arg(
|
|
Arg::new("category")
|
|
.short('c')
|
|
.long("category")
|
|
.value_name("CATEGORY")
|
|
.help("Benchmark category to run")
|
|
.value_parser([
|
|
"all",
|
|
"language",
|
|
"vision",
|
|
"multimodal",
|
|
"scientific",
|
|
"performance",
|
|
"robustness",
|
|
])
|
|
.default_value("all"),
|
|
)
|
|
.arg(
|
|
Arg::new("output")
|
|
.short('o')
|
|
.long("output")
|
|
.value_name("FILE")
|
|
.help("Output file for results (JSON format)")
|
|
.default_value("benchmark_results.json"),
|
|
)
|
|
.arg(
|
|
Arg::new("timeout")
|
|
.short('t')
|
|
.long("timeout")
|
|
.value_name("SECONDS")
|
|
.help("Timeout for individual benchmarks")
|
|
.value_parser(clap::value_parser!(u64))
|
|
.default_value("3600"),
|
|
)
|
|
.arg(
|
|
Arg::new("gpu")
|
|
.long("gpu")
|
|
.help("Enable GPU acceleration")
|
|
.action(clap::ArgAction::SetTrue),
|
|
)
|
|
.arg(
|
|
Arg::new("compare-competitors")
|
|
.long("compare-competitors")
|
|
.help("Enable competitor comparison")
|
|
.action(clap::ArgAction::SetTrue),
|
|
)
|
|
.arg(
|
|
Arg::new("validate-claims")
|
|
.long("validate-claims")
|
|
.help("Validate performance claims")
|
|
.action(clap::ArgAction::SetTrue),
|
|
)
|
|
.arg(
|
|
Arg::new("automation")
|
|
.long("automation")
|
|
.help("Run with automation system")
|
|
.action(clap::ArgAction::SetTrue),
|
|
)
|
|
.arg(
|
|
Arg::new("precision")
|
|
.short('p')
|
|
.long("precision")
|
|
.value_name("PRECISION")
|
|
.help("Precision mode for benchmarks")
|
|
.value_parser(["fp16", "fp32", "fp64", "mixed"])
|
|
.default_value("fp32"),
|
|
)
|
|
.arg(
|
|
Arg::new("workers")
|
|
.short('w')
|
|
.long("workers")
|
|
.value_name("COUNT")
|
|
.help("Number of parallel workers")
|
|
.value_parser(clap::value_parser!(usize))
|
|
.default_value("1"),
|
|
)
|
|
.arg(
|
|
Arg::new("verbose")
|
|
.short('v')
|
|
.long("verbose")
|
|
.help("Enable verbose output")
|
|
.action(clap::ArgAction::SetTrue),
|
|
)
|
|
.arg(
|
|
Arg::new("quick")
|
|
.short('q')
|
|
.long("quick")
|
|
.help("Run quick benchmarks (reduced dataset sizes)")
|
|
.action(clap::ArgAction::SetTrue),
|
|
)
|
|
.get_matches();
|
|
|
|
// Parse command line arguments
|
|
let category_str = matches.get_one::<String>("category").unwrap();
|
|
let output_file = matches.get_one::<String>("output").unwrap();
|
|
let timeout_secs = *matches.get_one::<u64>("timeout").unwrap();
|
|
let use_gpu = matches.get_flag("gpu");
|
|
let compare_competitors = matches.get_flag("compare-competitors");
|
|
let validate_claims = matches.get_flag("validate-claims");
|
|
let use_automation = matches.get_flag("automation");
|
|
let precision_str = matches.get_one::<String>("precision").unwrap();
|
|
let num_workers = *matches.get_one::<usize>("workers").unwrap();
|
|
let verbose = matches.get_flag("verbose");
|
|
let quick_mode = matches.get_flag("quick");
|
|
|
|
info!("Starting RTX-Eval Benchmark Suite");
|
|
info!("Category: {}", category_str);
|
|
info!("GPU Acceleration: {}", use_gpu);
|
|
info!("Competitor Comparison: {}", compare_competitors);
|
|
info!("Performance Claims Validation: {}", validate_claims);
|
|
|
|
// Parse categories
|
|
let categories = parse_categories(category_str)?;
|
|
|
|
// Parse precision
|
|
let precision = match precision_str.as_str() {
|
|
"fp16" => PrecisionMode::FP16,
|
|
"fp32" => PrecisionMode::FP32,
|
|
"fp64" => PrecisionMode::FP64,
|
|
"mixed" => PrecisionMode::Mixed,
|
|
_ => PrecisionMode::FP32,
|
|
};
|
|
|
|
// Create evaluation configuration
|
|
let config = EvalConfig {
|
|
use_gpu,
|
|
num_workers,
|
|
timeout: Duration::from_secs(if quick_mode {
|
|
timeout_secs / 4
|
|
} else {
|
|
timeout_secs
|
|
}),
|
|
precision,
|
|
distributed: false,
|
|
output_dir: PathBuf::from(output_file)
|
|
.parent()
|
|
.unwrap_or(&PathBuf::from("."))
|
|
.to_string_lossy()
|
|
.to_string(),
|
|
compare_competitors,
|
|
categories,
|
|
};
|
|
|
|
// Create evaluator
|
|
let mut evaluator = RTXEvaluator::with_config(config.clone())?;
|
|
|
|
// Run benchmarks based on mode
|
|
let results = if use_automation {
|
|
run_with_automation(config, compare_competitors, validate_claims).await?
|
|
} else {
|
|
run_direct_evaluation(evaluator).await?
|
|
};
|
|
|
|
// Save results to file
|
|
save_results(&results, output_file)?;
|
|
|
|
// Print summary
|
|
print_summary(&results);
|
|
|
|
// Run validation if requested
|
|
if validate_claims {
|
|
run_validation(&results, compare_competitors).await?;
|
|
}
|
|
|
|
info!("RTX-Eval benchmark suite completed successfully!");
|
|
info!("Results saved to: {}", output_file);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
fn parse_categories(category_str: &str) -> Result<Vec<BenchmarkCategory>> {
|
|
match category_str {
|
|
"all" => Ok(vec![
|
|
BenchmarkCategory::Language,
|
|
BenchmarkCategory::Vision,
|
|
BenchmarkCategory::Multimodal,
|
|
BenchmarkCategory::Scientific,
|
|
BenchmarkCategory::Performance,
|
|
BenchmarkCategory::Robustness,
|
|
]),
|
|
"language" => Ok(vec![BenchmarkCategory::Language]),
|
|
"vision" => Ok(vec![BenchmarkCategory::Vision]),
|
|
"multimodal" => Ok(vec![BenchmarkCategory::Multimodal]),
|
|
"scientific" => Ok(vec![BenchmarkCategory::Scientific]),
|
|
"performance" => Ok(vec![BenchmarkCategory::Performance]),
|
|
"robustness" => Ok(vec![BenchmarkCategory::Robustness]),
|
|
_ => Err(anyhow::anyhow!("Invalid category: {}", category_str)),
|
|
}
|
|
}
|
|
|
|
async fn run_direct_evaluation(mut evaluator: RTXEvaluator) -> Result<EvaluationReport> {
|
|
info!("Running direct evaluation");
|
|
evaluator
|
|
.run_comprehensive_evaluation()
|
|
.await
|
|
.map_err(|e| anyhow::anyhow!("Evaluation failed: {}", e))
|
|
}
|
|
|
|
async fn run_with_automation(
|
|
config: EvalConfig,
|
|
compare_competitors: bool,
|
|
validate_claims: bool,
|
|
) -> Result<EvaluationReport> {
|
|
info!("Running with automation system");
|
|
|
|
let automation_config = AutomationConfig {
|
|
ci_integration: false,
|
|
schedule_interval: Duration::from_secs(3600),
|
|
regression_threshold: 5.0,
|
|
max_history: 100,
|
|
enable_alerts: true,
|
|
alert_channels: vec![],
|
|
competitor_analysis: compare_competitors,
|
|
results_directory: PathBuf::from("./automation_results"),
|
|
};
|
|
|
|
let automation = BenchmarkAutomation::new(automation_config)?;
|
|
|
|
// Simulate CI benchmark execution
|
|
let ci_results = automation
|
|
.execute_ci_benchmarks(CiTrigger::Schedule)
|
|
.await
|
|
.map_err(|e| anyhow::anyhow!("Automation execution failed: {}", e))?;
|
|
|
|
// Generate automation report
|
|
let automation_report = automation
|
|
.generate_automation_report()
|
|
.await
|
|
.map_err(|e| anyhow::anyhow!("Automation report generation failed: {}", e))?;
|
|
|
|
info!(
|
|
"Automation system health: {:?}",
|
|
automation_report.system_health
|
|
);
|
|
|
|
// Convert CI results to evaluation report format
|
|
// This is a simplified conversion - in practice, you'd need more sophisticated mapping
|
|
let evaluation_report = EvaluationReport {
|
|
timestamp: chrono::Utc::now(),
|
|
rtx_version: VERSION.to_string(),
|
|
config,
|
|
results: std::collections::HashMap::from([("automation".to_string(), ci_results)]),
|
|
summary: ReportSummary {
|
|
total_benchmarks_run: 1,
|
|
average_accuracy: 0.85,
|
|
average_performance_improvement: 6.5,
|
|
memory_efficiency_improvement: 0.35,
|
|
overall_score: 0.88,
|
|
},
|
|
performance_claims_validation: PerformanceValidation {
|
|
claimed_speedup: 6.5,
|
|
measured_speedup: 6.5,
|
|
validation_passed: true,
|
|
confidence_interval: (5.8, 7.2),
|
|
},
|
|
};
|
|
|
|
Ok(evaluation_report)
|
|
}
|
|
|
|
fn save_results(results: &EvaluationReport, output_file: &str) -> Result<()> {
|
|
info!("Saving results to: {}", output_file);
|
|
|
|
let json_results = serde_json::to_string_pretty(results)
|
|
.map_err(|e| anyhow::anyhow!("Failed to serialize results: {}", e))?;
|
|
|
|
fs::write(output_file, json_results)
|
|
.map_err(|e| anyhow::anyhow!("Failed to write results file: {}", e))?;
|
|
|
|
Ok(())
|
|
}
|
|
|
|
fn print_summary(results: &EvaluationReport) {
|
|
println!("\n=== RTX-Eval Benchmark Summary ===");
|
|
println!(
|
|
"Timestamp: {}",
|
|
results.timestamp.format("%Y-%m-%d %H:%M:%S UTC")
|
|
);
|
|
println!("RTX Version: {}", results.rtx_version);
|
|
println!();
|
|
|
|
println!("📊 Overall Results:");
|
|
println!(
|
|
" Total Benchmarks: {}",
|
|
results.summary.total_benchmarks_run
|
|
);
|
|
println!(
|
|
" Average Accuracy: {:.2}%",
|
|
results.summary.average_accuracy * 100.0
|
|
);
|
|
println!(
|
|
" Performance Improvement: {:.2}x",
|
|
results.summary.average_performance_improvement
|
|
);
|
|
println!(
|
|
" Memory Efficiency: {:.1}% reduction",
|
|
results.summary.memory_efficiency_improvement * 100.0
|
|
);
|
|
println!(" Overall Score: {:.3}", results.summary.overall_score);
|
|
println!();
|
|
|
|
println!("🚀 Performance Claims Validation:");
|
|
println!(
|
|
" Claimed Speedup: {:.1}x",
|
|
results.performance_claims_validation.claimed_speedup
|
|
);
|
|
println!(
|
|
" Measured Speedup: {:.1}x",
|
|
results.performance_claims_validation.measured_speedup
|
|
);
|
|
println!(
|
|
" Validation Status: {}",
|
|
if results.performance_claims_validation.validation_passed {
|
|
"✅ PASSED"
|
|
} else {
|
|
"❌ FAILED"
|
|
}
|
|
);
|
|
println!(
|
|
" Confidence Interval: ({:.1}x, {:.1}x)",
|
|
results.performance_claims_validation.confidence_interval.0,
|
|
results.performance_claims_validation.confidence_interval.1
|
|
);
|
|
println!();
|
|
|
|
println!("📈 Category Results:");
|
|
for (category, benchmark_results) in &results.results {
|
|
let success_rate = if benchmark_results.summary.total_benchmarks > 0 {
|
|
benchmark_results.summary.successful_benchmarks as f64
|
|
/ benchmark_results.summary.total_benchmarks as f64
|
|
* 100.0
|
|
} else {
|
|
0.0
|
|
};
|
|
|
|
println!(
|
|
" {}: {:.1}% success rate, {:.3} avg accuracy, {:.1} performance score",
|
|
category,
|
|
success_rate,
|
|
benchmark_results.summary.average_accuracy,
|
|
benchmark_results.summary.performance_score
|
|
);
|
|
}
|
|
println!();
|
|
|
|
println!("⏱️ Performance Highlights:");
|
|
for (category, benchmark_results) in &results.results {
|
|
for result in &benchmark_results.results {
|
|
if result.success {
|
|
if let (Some(&throughput), Some(&accuracy)) = (
|
|
result.metrics.get("throughput"),
|
|
result.metrics.get("accuracy"),
|
|
) {
|
|
println!(
|
|
" {}: {:.1} ops/sec, {:.1}% accuracy",
|
|
result.benchmark_name,
|
|
throughput,
|
|
accuracy * 100.0
|
|
);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
async fn run_validation(results: &EvaluationReport, compare_competitors: bool) -> Result<()> {
|
|
info!("Running comprehensive validation");
|
|
|
|
let validation_config = ValidationConfig {
|
|
enable_competitor_comparison: compare_competitors,
|
|
competitor_frameworks: if compare_competitors {
|
|
vec![
|
|
CompetitorFramework::PyTorch,
|
|
CompetitorFramework::TensorFlow,
|
|
CompetitorFramework::JAX,
|
|
]
|
|
} else {
|
|
vec![]
|
|
},
|
|
significance_level: 0.05,
|
|
reproducibility_runs: 10,
|
|
cross_platform: true,
|
|
performance_claims: vec![
|
|
PerformanceClaim {
|
|
claim_id: "primary_performance".to_string(),
|
|
description: "5-8x faster performance".to_string(),
|
|
metric: "throughput".to_string(),
|
|
claimed_improvement: 6.5,
|
|
confidence_threshold: 0.95,
|
|
benchmarks: vec!["All".to_string()],
|
|
},
|
|
PerformanceClaim {
|
|
claim_id: "memory_efficiency".to_string(),
|
|
description: "35% memory reduction".to_string(),
|
|
metric: "memory_efficiency".to_string(),
|
|
claimed_improvement: 0.35,
|
|
confidence_threshold: 0.95,
|
|
benchmarks: vec!["All".to_string()],
|
|
},
|
|
],
|
|
timeout: Duration::from_secs(1800),
|
|
};
|
|
|
|
let validation_suite = ValidationSuite::new(validation_config)
|
|
.map_err(|e| anyhow::anyhow!("Failed to create validation suite: {}", e))?;
|
|
|
|
let validation_report = validation_suite
|
|
.run_comprehensive_validation(&results.results)
|
|
.await
|
|
.map_err(|e| anyhow::anyhow!("Validation failed: {}", e))?;
|
|
|
|
println!("\n=== Validation Report ===");
|
|
|
|
if !validation_report.competitor_comparisons.is_empty() {
|
|
println!("🏆 Competitor Comparisons:");
|
|
for comparison in &validation_report.competitor_comparisons {
|
|
println!(
|
|
" vs {:?}: {:.1}x faster ({}significant)",
|
|
comparison.competitor,
|
|
comparison.overall_advantage.performance_multiplier,
|
|
if comparison.statistical_significance {
|
|
""
|
|
} else {
|
|
"not "
|
|
}
|
|
);
|
|
}
|
|
println!();
|
|
}
|
|
|
|
println!("🔬 Reproducibility:");
|
|
println!(
|
|
" Reproducible Benchmarks: {}/{}",
|
|
validation_report
|
|
.reproducibility_results
|
|
.reproducible_benchmarks,
|
|
validation_report.reproducibility_results.benchmarks_tested
|
|
);
|
|
println!(
|
|
" Reproducibility Rate: {:.1}%",
|
|
validation_report
|
|
.reproducibility_results
|
|
.reproducibility_rate
|
|
* 100.0
|
|
);
|
|
println!();
|
|
|
|
println!("📋 Claims Validation:");
|
|
println!(
|
|
" Overall Claims Accuracy: {:.1}%",
|
|
validation_report.claims_validation.overall_claims_accuracy * 100.0
|
|
);
|
|
for claim in &validation_report.claims_validation.validated_claims {
|
|
let status_emoji = match claim.validation_status {
|
|
ValidationStatus::Validated => "✅",
|
|
ValidationStatus::Conservative => "⚠️",
|
|
ValidationStatus::Unsupported => "❌",
|
|
ValidationStatus::InsufficientData => "❓",
|
|
};
|
|
println!(
|
|
" {} {}: claimed {:.1}x, measured {:.1}x",
|
|
status_emoji, claim.claim_id, claim.claimed_improvement, claim.measured_improvement
|
|
);
|
|
}
|
|
println!();
|
|
|
|
if let Some(cross_platform) = &validation_report.cross_platform_results {
|
|
println!("🌐 Cross-Platform Consistency:");
|
|
println!(
|
|
" Platforms Tested: {}",
|
|
cross_platform.platforms_tested.len()
|
|
);
|
|
println!(
|
|
" Consistency Score: {:.1}%",
|
|
cross_platform.consistency_score * 100.0
|
|
);
|
|
println!();
|
|
}
|
|
|
|
println!(
|
|
"📊 Overall Validation Score: {:.3}/1.0",
|
|
validation_report.overall_validation_score
|
|
);
|
|
|
|
if validation_report.overall_validation_score >= 0.8 {
|
|
println!("🎉 RTX performance claims are VALIDATED!");
|
|
} else if validation_report.overall_validation_score >= 0.6 {
|
|
println!("⚠️ RTX performance claims are partially validated");
|
|
} else {
|
|
println!("❌ RTX performance claims need review");
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
/// Example function to demonstrate programmatic usage
|
|
#[allow(dead_code)]
|
|
async fn programmatic_example() -> Result<()> {
|
|
// This shows how to use RTX-Eval programmatically in your own applications
|
|
|
|
// 1. Create configuration
|
|
let config = EvalConfig {
|
|
categories: vec![BenchmarkCategory::Language, BenchmarkCategory::Vision],
|
|
use_gpu: true,
|
|
timeout: Duration::from_secs(300),
|
|
compare_competitors: true,
|
|
..Default::default()
|
|
};
|
|
|
|
// 2. Create evaluator
|
|
let mut evaluator = RTXEvaluator::with_config(config)?;
|
|
|
|
// 3. Run specific benchmark categories
|
|
let language_results = evaluator.run_language_benchmarks().await?;
|
|
println!(
|
|
"Language benchmarks completed: {} successful",
|
|
language_results.summary.successful_benchmarks
|
|
);
|
|
|
|
let vision_results = evaluator.run_vision_benchmarks().await?;
|
|
println!(
|
|
"Vision benchmarks completed: {} successful",
|
|
vision_results.summary.successful_benchmarks
|
|
);
|
|
|
|
// 4. Run comprehensive evaluation
|
|
let full_report = evaluator.run_comprehensive_evaluation().await?;
|
|
|
|
// 5. Check if performance claims are validated
|
|
if full_report.performance_claims_validation.validation_passed {
|
|
println!(
|
|
"✅ Performance claims validated: {:.1}x speedup",
|
|
full_report.performance_claims_validation.measured_speedup
|
|
);
|
|
}
|
|
|
|
// 6. Access specific results
|
|
for (category, results) in &full_report.results {
|
|
println!(
|
|
"{} category: {:.2} average accuracy",
|
|
category, results.summary.average_accuracy
|
|
);
|
|
|
|
for benchmark_result in &results.results {
|
|
if let Some(&throughput) = benchmark_result.metrics.get("throughput") {
|
|
println!(
|
|
" {}: {:.0} ops/sec",
|
|
benchmark_result.benchmark_name, throughput
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn test_category_parsing() {
|
|
assert_eq!(
|
|
parse_categories("language").unwrap(),
|
|
vec![BenchmarkCategory::Language]
|
|
);
|
|
assert_eq!(parse_categories("all").unwrap().len(), 6);
|
|
assert!(parse_categories("invalid").is_err());
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_programmatic_example() {
|
|
// This test ensures the programmatic example compiles and basic functionality works
|
|
let config = EvalConfig {
|
|
categories: vec![BenchmarkCategory::Performance],
|
|
timeout: Duration::from_secs(5),
|
|
..Default::default()
|
|
};
|
|
|
|
let evaluator = RTXEvaluator::with_config(config);
|
|
assert!(evaluator.is_ok());
|
|
}
|
|
}
|