osobhandClaude Opus 4.6
02d382d5f6
Consistent formatting pass: line wrapping, import sorting, trailing whitespace removal, let-chain indentation, merged derive attributes, and unsafe block reformatting. Co-Authored-By: Claude Opus 4.6 (1M context) <[email protected]>
910 lines
30 KiB
Rust
910 lines
30 KiB
Rust
//! Backend Performance Benchmarking for Distributed Training
|
|
//!
|
|
//! This module provides infrastructure for comparing performance and
|
|
//! feature parity between different communication backends (NCCL, RCCL, TCP).
|
|
//!
|
|
//! # Features
|
|
//!
|
|
//! - Latency and bandwidth measurements for all collective operations
|
|
//! - Automatic NCCL vs RCCL parity verification
|
|
//! - Customizable message sizes and iteration counts
|
|
//! - Report generation in multiple formats
|
|
//!
|
|
//! # Example
|
|
//!
|
|
//! ```rust,ignore
|
|
//! use rtx_distributed::backend_benchmark::{BackendBenchmark, BenchmarkConfig};
|
|
//!
|
|
//! let config = BenchmarkConfig::default();
|
|
//! let mut benchmark = BackendBenchmark::new(config);
|
|
//! let results = benchmark.run().await?;
|
|
//! let report = benchmark.generate_report();
|
|
//! println!("{}", report);
|
|
//! ```
|
|
|
|
use crate::backend::Backend;
|
|
use crate::collective_fusion::CollectiveType;
|
|
use crate::error::{DistributedError, Result};
|
|
use parking_lot::RwLock;
|
|
use serde::{Deserialize, Serialize};
|
|
use std::collections::HashMap;
|
|
use std::sync::Arc;
|
|
use std::time::{Duration, Instant};
|
|
|
|
// =============================================================================
|
|
// Benchmark Configuration
|
|
// =============================================================================
|
|
|
|
/// Configuration for the benchmark suite
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct BenchmarkConfig {
|
|
/// Message sizes to test (in bytes)
|
|
pub message_sizes: Vec<usize>,
|
|
/// Collective operations to benchmark
|
|
pub operations: Vec<CollectiveType>,
|
|
/// Number of iterations per benchmark
|
|
pub iterations: usize,
|
|
/// Warmup iterations before measurement
|
|
pub warmup_iterations: usize,
|
|
/// Backends to benchmark
|
|
pub backends: Vec<Backend>,
|
|
/// Number of GPU devices to use
|
|
pub num_gpus: usize,
|
|
/// Enable verification of results
|
|
pub verify_results: bool,
|
|
/// Timeout per operation in seconds
|
|
pub timeout_secs: u64,
|
|
}
|
|
|
|
impl Default for BenchmarkConfig {
|
|
fn default() -> Self {
|
|
Self {
|
|
message_sizes: vec![
|
|
1024, // 1 KB
|
|
1024 * 1024, // 1 MB
|
|
10 * 1024 * 1024, // 10 MB
|
|
100 * 1024 * 1024, // 100 MB
|
|
],
|
|
operations: vec![
|
|
CollectiveType::AllReduce,
|
|
CollectiveType::AllGather,
|
|
CollectiveType::ReduceScatter,
|
|
CollectiveType::Broadcast,
|
|
],
|
|
iterations: 100,
|
|
warmup_iterations: 10,
|
|
backends: vec![Backend::Nccl, Backend::Rccl, Backend::Tcp],
|
|
num_gpus: 2,
|
|
verify_results: true,
|
|
timeout_secs: 60,
|
|
}
|
|
}
|
|
}
|
|
|
|
impl BenchmarkConfig {
|
|
/// Create config for quick benchmarks
|
|
pub fn quick() -> Self {
|
|
Self {
|
|
message_sizes: vec![1024, 1024 * 1024],
|
|
iterations: 10,
|
|
warmup_iterations: 2,
|
|
..Default::default()
|
|
}
|
|
}
|
|
|
|
/// Create config for comprehensive benchmarks
|
|
pub fn comprehensive() -> Self {
|
|
Self {
|
|
message_sizes: vec![
|
|
1024, // 1 KB
|
|
4 * 1024, // 4 KB
|
|
16 * 1024, // 16 KB
|
|
64 * 1024, // 64 KB
|
|
256 * 1024, // 256 KB
|
|
1024 * 1024, // 1 MB
|
|
4 * 1024 * 1024, // 4 MB
|
|
16 * 1024 * 1024, // 16 MB
|
|
64 * 1024 * 1024, // 64 MB
|
|
256 * 1024 * 1024, // 256 MB
|
|
1024 * 1024 * 1024, // 1 GB
|
|
],
|
|
operations: vec![
|
|
CollectiveType::AllReduce,
|
|
CollectiveType::AllGather,
|
|
CollectiveType::ReduceScatter,
|
|
CollectiveType::Broadcast,
|
|
CollectiveType::Reduce,
|
|
CollectiveType::Scatter,
|
|
CollectiveType::Gather,
|
|
],
|
|
iterations: 1000,
|
|
warmup_iterations: 100,
|
|
..Default::default()
|
|
}
|
|
}
|
|
|
|
/// Create config to compare NCCL and RCCL only
|
|
pub fn nccl_vs_rccl() -> Self {
|
|
Self {
|
|
backends: vec![Backend::Nccl, Backend::Rccl],
|
|
..Default::default()
|
|
}
|
|
}
|
|
}
|
|
|
|
// =============================================================================
|
|
// Benchmark Results
|
|
// =============================================================================
|
|
|
|
/// Single benchmark measurement result
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct BenchmarkResult {
|
|
/// Backend used for this measurement
|
|
pub backend: Backend,
|
|
/// Collective operation tested
|
|
pub operation: CollectiveType,
|
|
/// Message size in bytes
|
|
pub message_size: usize,
|
|
/// Latency statistics
|
|
pub latency: LatencyStats,
|
|
/// Bandwidth achieved (GB/s)
|
|
pub bandwidth_gbps: f64,
|
|
/// Number of iterations performed
|
|
pub iterations: usize,
|
|
/// Whether the operation succeeded
|
|
pub success: bool,
|
|
/// Error message if failed
|
|
pub error: Option<String>,
|
|
}
|
|
|
|
/// Latency statistics for a benchmark
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct LatencyStats {
|
|
/// Minimum latency in microseconds
|
|
pub min_us: f64,
|
|
/// Maximum latency in microseconds
|
|
pub max_us: f64,
|
|
/// Mean latency in microseconds
|
|
pub mean_us: f64,
|
|
/// Median latency in microseconds
|
|
pub median_us: f64,
|
|
/// 99th percentile latency
|
|
pub p99_us: f64,
|
|
/// Standard deviation
|
|
pub std_dev_us: f64,
|
|
}
|
|
|
|
impl LatencyStats {
|
|
/// Compute statistics from a slice of latencies
|
|
pub fn from_measurements(latencies: &[f64]) -> Self {
|
|
if latencies.is_empty() {
|
|
return Self {
|
|
min_us: 0.0,
|
|
max_us: 0.0,
|
|
mean_us: 0.0,
|
|
median_us: 0.0,
|
|
p99_us: 0.0,
|
|
std_dev_us: 0.0,
|
|
};
|
|
}
|
|
|
|
let mut sorted = latencies.to_vec();
|
|
sorted.sort_by(|a, b| a.partial_cmp(b).unwrap());
|
|
|
|
let min = sorted[0];
|
|
let max = sorted[sorted.len() - 1];
|
|
let mean: f64 = latencies.iter().sum::<f64>() / latencies.len() as f64;
|
|
|
|
let median = if sorted.len() % 2 == 0 {
|
|
f64::midpoint(sorted[sorted.len() / 2 - 1], sorted[sorted.len() / 2])
|
|
} else {
|
|
sorted[sorted.len() / 2]
|
|
};
|
|
|
|
let p99_idx = ((sorted.len() as f64) * 0.99) as usize;
|
|
let p99 = sorted[p99_idx.min(sorted.len() - 1)];
|
|
|
|
let variance: f64 =
|
|
latencies.iter().map(|x| (x - mean).powi(2)).sum::<f64>() / latencies.len() as f64;
|
|
let std_dev = variance.sqrt();
|
|
|
|
Self {
|
|
min_us: min,
|
|
max_us: max,
|
|
mean_us: mean,
|
|
median_us: median,
|
|
p99_us: p99,
|
|
std_dev_us: std_dev,
|
|
}
|
|
}
|
|
}
|
|
|
|
// =============================================================================
|
|
// Parity Report
|
|
// =============================================================================
|
|
|
|
/// Feature parity report between backends
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct ParityReport {
|
|
/// Backends compared
|
|
pub backends: Vec<Backend>,
|
|
/// Parity results for each operation
|
|
pub operation_parity: HashMap<CollectiveType, OperationParity>,
|
|
/// Overall parity percentage
|
|
pub overall_parity_percent: f64,
|
|
/// List of missing features
|
|
pub missing_features: Vec<ParityGap>,
|
|
/// Performance comparison
|
|
pub performance_comparison: PerformanceComparison,
|
|
}
|
|
|
|
/// Parity status for a specific operation
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct OperationParity {
|
|
/// Operation type
|
|
pub operation: CollectiveType,
|
|
/// Is the operation supported by all backends?
|
|
pub supported_all: bool,
|
|
/// Backends that support this operation
|
|
pub supported_by: Vec<Backend>,
|
|
/// Is numerical output equivalent?
|
|
pub numerically_equivalent: bool,
|
|
/// Max relative error observed
|
|
pub max_relative_error: f64,
|
|
}
|
|
|
|
/// Gap in feature parity
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct ParityGap {
|
|
/// Feature that's missing
|
|
pub feature: String,
|
|
/// Backend that's missing the feature
|
|
pub missing_in: Backend,
|
|
/// Backend that has the feature
|
|
pub present_in: Backend,
|
|
/// Severity (low, medium, high)
|
|
pub severity: String,
|
|
}
|
|
|
|
/// Performance comparison between backends
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct PerformanceComparison {
|
|
/// Relative performance for each operation (normalized to fastest)
|
|
pub relative_performance: HashMap<(Backend, CollectiveType), f64>,
|
|
/// Fastest backend for each operation
|
|
pub fastest_for: HashMap<CollectiveType, Backend>,
|
|
/// Overall performance ranking
|
|
pub overall_ranking: Vec<(Backend, f64)>,
|
|
}
|
|
|
|
// =============================================================================
|
|
// Benchmark Report
|
|
// =============================================================================
|
|
|
|
/// Complete benchmark report
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct BenchmarkReport {
|
|
/// Benchmark configuration
|
|
pub config: BenchmarkConfig,
|
|
/// Individual results
|
|
pub results: Vec<BenchmarkResult>,
|
|
/// Aggregated by operation type
|
|
pub by_operation: HashMap<CollectiveType, Vec<BenchmarkResult>>,
|
|
/// Aggregated by backend
|
|
pub by_backend: HashMap<Backend, Vec<BenchmarkResult>>,
|
|
/// Summary statistics
|
|
pub summary: BenchmarkSummary,
|
|
/// Parity report (if multiple backends)
|
|
pub parity: Option<ParityReport>,
|
|
/// Timestamp
|
|
pub timestamp: String,
|
|
}
|
|
|
|
/// Summary statistics across all benchmarks
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct BenchmarkSummary {
|
|
/// Total benchmarks run
|
|
pub total_benchmarks: usize,
|
|
/// Successful benchmarks
|
|
pub successful: usize,
|
|
/// Failed benchmarks
|
|
pub failed: usize,
|
|
/// Total time spent
|
|
pub total_duration_secs: f64,
|
|
/// Best bandwidth achieved
|
|
pub best_bandwidth_gbps: f64,
|
|
/// Best bandwidth details
|
|
pub best_bandwidth_details: String,
|
|
/// Worst latency observed
|
|
pub worst_latency_us: f64,
|
|
}
|
|
|
|
// =============================================================================
|
|
// Backend Benchmark Suite
|
|
// =============================================================================
|
|
|
|
/// Backend performance benchmark suite
|
|
#[derive(Debug)]
|
|
pub struct BackendBenchmark {
|
|
/// Benchmark configuration
|
|
config: BenchmarkConfig,
|
|
/// Collected results
|
|
results: RwLock<Vec<BenchmarkResult>>,
|
|
/// Start time
|
|
start_time: Option<Instant>,
|
|
}
|
|
|
|
impl BackendBenchmark {
|
|
/// Create a new benchmark suite
|
|
pub fn new(config: BenchmarkConfig) -> Self {
|
|
Self {
|
|
config,
|
|
results: RwLock::new(Vec::new()),
|
|
start_time: None,
|
|
}
|
|
}
|
|
|
|
/// Run the complete benchmark suite
|
|
pub async fn run(&mut self) -> Result<Vec<BenchmarkResult>> {
|
|
self.start_time = Some(Instant::now());
|
|
let mut all_results = Vec::new();
|
|
|
|
for backend in &self.config.backends.clone() {
|
|
for operation in &self.config.operations.clone() {
|
|
for &message_size in &self.config.message_sizes.clone() {
|
|
match self
|
|
.run_single_benchmark(*backend, *operation, message_size)
|
|
.await
|
|
{
|
|
Ok(result) => {
|
|
all_results.push(result.clone());
|
|
self.results.write().push(result);
|
|
}
|
|
Err(e) => {
|
|
let failed_result = BenchmarkResult {
|
|
backend: *backend,
|
|
operation: *operation,
|
|
message_size,
|
|
latency: LatencyStats::from_measurements(&[]),
|
|
bandwidth_gbps: 0.0,
|
|
iterations: 0,
|
|
success: false,
|
|
error: Some(e.to_string()),
|
|
};
|
|
all_results.push(failed_result.clone());
|
|
self.results.write().push(failed_result);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
Ok(all_results)
|
|
}
|
|
|
|
/// Run a single benchmark
|
|
async fn run_single_benchmark(
|
|
&self,
|
|
backend: Backend,
|
|
operation: CollectiveType,
|
|
message_size: usize,
|
|
) -> Result<BenchmarkResult> {
|
|
// Simulate benchmark execution
|
|
// In real implementation, this would:
|
|
// 1. Initialize the backend
|
|
// 2. Allocate GPU memory
|
|
// 3. Run warmup iterations
|
|
// 4. Run timed iterations
|
|
// 5. Collect timing data
|
|
|
|
let mut latencies = Vec::with_capacity(self.config.iterations);
|
|
|
|
// Simulated warmup
|
|
for _ in 0..self.config.warmup_iterations {
|
|
self.simulate_collective(backend, operation, message_size)?;
|
|
}
|
|
|
|
// Timed iterations
|
|
for _ in 0..self.config.iterations {
|
|
let start = Instant::now();
|
|
self.simulate_collective(backend, operation, message_size)?;
|
|
let elapsed = start.elapsed();
|
|
latencies.push(elapsed.as_secs_f64() * 1_000_000.0); // Convert to microseconds
|
|
}
|
|
|
|
let latency_stats = LatencyStats::from_measurements(&latencies);
|
|
|
|
// Calculate bandwidth: bytes transferred / time
|
|
// For AllReduce, 2 * (n-1) * data_size / n where n is world size
|
|
let bytes_transferred = match operation {
|
|
CollectiveType::AllReduce => 2.0 * message_size as f64,
|
|
CollectiveType::AllGather => message_size as f64 * self.config.num_gpus as f64,
|
|
CollectiveType::ReduceScatter => message_size as f64,
|
|
CollectiveType::Broadcast => message_size as f64,
|
|
_ => message_size as f64,
|
|
};
|
|
|
|
let bandwidth_gbps = bytes_transferred / (latency_stats.mean_us / 1_000_000.0) / 1e9;
|
|
|
|
Ok(BenchmarkResult {
|
|
backend,
|
|
operation,
|
|
message_size,
|
|
latency: latency_stats,
|
|
bandwidth_gbps,
|
|
iterations: self.config.iterations,
|
|
success: true,
|
|
error: None,
|
|
})
|
|
}
|
|
|
|
/// Simulate a collective operation (placeholder for actual implementation)
|
|
fn simulate_collective(
|
|
&self,
|
|
backend: Backend,
|
|
operation: CollectiveType,
|
|
message_size: usize,
|
|
) -> Result<()> {
|
|
// Base latency varies by backend
|
|
let base_latency_us = match backend {
|
|
Backend::Nccl => 5.0,
|
|
Backend::Rccl => 6.0, // Slightly higher for RCCL
|
|
Backend::Rnccl => 5.5, // Rust NCCL, similar to NCCL
|
|
Backend::Tcp => 100.0,
|
|
Backend::Cpu => 50.0,
|
|
Backend::Mpi => 20.0,
|
|
};
|
|
|
|
// Operation overhead
|
|
let op_overhead = match operation {
|
|
CollectiveType::AllReduce => 1.5,
|
|
CollectiveType::AllGather => 1.2,
|
|
CollectiveType::ReduceScatter => 1.3,
|
|
CollectiveType::Broadcast => 1.0,
|
|
_ => 1.0,
|
|
};
|
|
|
|
// Size-dependent latency (bandwidth limited)
|
|
let bandwidth_gbps = match backend {
|
|
Backend::Nccl => 300.0, // NVLink bandwidth
|
|
Backend::Rccl => 200.0, // Infinity Fabric bandwidth
|
|
Backend::Rnccl => 300.0, // Rust NCCL, similar to NVLink
|
|
Backend::Tcp => 25.0, // 100GbE
|
|
Backend::Cpu => 50.0,
|
|
Backend::Mpi => 100.0,
|
|
};
|
|
|
|
let transfer_time_us = (message_size as f64 / (bandwidth_gbps * 1e9)) * 1_000_000.0;
|
|
let total_time_us = base_latency_us * op_overhead + transfer_time_us;
|
|
|
|
// Simulate the time
|
|
let sleep_time = Duration::from_nanos((total_time_us * 1000.0) as u64);
|
|
std::thread::sleep(sleep_time);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
/// Generate a comprehensive benchmark report
|
|
pub fn generate_report(&self) -> BenchmarkReport {
|
|
let results = self.results.read().clone();
|
|
|
|
// Group by operation
|
|
let mut by_operation: HashMap<CollectiveType, Vec<BenchmarkResult>> = HashMap::new();
|
|
for result in &results {
|
|
by_operation
|
|
.entry(result.operation)
|
|
.or_default()
|
|
.push(result.clone());
|
|
}
|
|
|
|
// Group by backend
|
|
let mut by_backend: HashMap<Backend, Vec<BenchmarkResult>> = HashMap::new();
|
|
for result in &results {
|
|
by_backend
|
|
.entry(result.backend)
|
|
.or_default()
|
|
.push(result.clone());
|
|
}
|
|
|
|
// Compute summary
|
|
let successful = results.iter().filter(|r| r.success).count();
|
|
let failed = results.len() - successful;
|
|
|
|
let best_bw = results
|
|
.iter()
|
|
.filter(|r| r.success)
|
|
.max_by(|a, b| a.bandwidth_gbps.partial_cmp(&b.bandwidth_gbps).unwrap());
|
|
|
|
let worst_latency = results
|
|
.iter()
|
|
.filter(|r| r.success)
|
|
.map(|r| r.latency.p99_us)
|
|
.fold(0.0f64, f64::max);
|
|
|
|
let summary = BenchmarkSummary {
|
|
total_benchmarks: results.len(),
|
|
successful,
|
|
failed,
|
|
total_duration_secs: self.start_time.map_or(0.0, |t| t.elapsed().as_secs_f64()),
|
|
best_bandwidth_gbps: best_bw.map_or(0.0, |r| r.bandwidth_gbps),
|
|
best_bandwidth_details: best_bw
|
|
.map(|r| format!("{:?} {:?} {}B", r.backend, r.operation, r.message_size))
|
|
.unwrap_or_default(),
|
|
worst_latency_us: worst_latency,
|
|
};
|
|
|
|
// Generate parity report if multiple backends
|
|
let parity = if self.config.backends.len() > 1 {
|
|
Some(self.verify_parity())
|
|
} else {
|
|
None
|
|
};
|
|
|
|
BenchmarkReport {
|
|
config: self.config.clone(),
|
|
results,
|
|
by_operation,
|
|
by_backend,
|
|
summary,
|
|
parity,
|
|
timestamp: chrono::Utc::now().to_rfc3339(),
|
|
}
|
|
}
|
|
|
|
/// Verify feature parity between backends
|
|
pub fn verify_parity(&self) -> ParityReport {
|
|
let results = self.results.read();
|
|
let backends = self.config.backends.clone();
|
|
|
|
let mut operation_parity = HashMap::new();
|
|
let mut missing_features = Vec::new();
|
|
let mut relative_performance: HashMap<(Backend, CollectiveType), f64> = HashMap::new();
|
|
let mut fastest_for: HashMap<CollectiveType, Backend> = HashMap::new();
|
|
|
|
// Check parity for each operation
|
|
for op in &self.config.operations {
|
|
let op_results: Vec<_> = results
|
|
.iter()
|
|
.filter(|r| r.operation == *op && r.success)
|
|
.collect();
|
|
|
|
let supported_by: Vec<Backend> = op_results
|
|
.iter()
|
|
.map(|r| r.backend)
|
|
.collect::<std::collections::HashSet<_>>()
|
|
.into_iter()
|
|
.collect();
|
|
|
|
// Find fastest backend for this operation
|
|
if let Some(fastest) = op_results
|
|
.iter()
|
|
.filter(|r| r.success)
|
|
.min_by(|a, b| a.latency.mean_us.partial_cmp(&b.latency.mean_us).unwrap())
|
|
{
|
|
fastest_for.insert(*op, fastest.backend);
|
|
|
|
// Calculate relative performance
|
|
let best_time = fastest.latency.mean_us;
|
|
for result in &op_results {
|
|
let relative = best_time / result.latency.mean_us;
|
|
relative_performance.insert((result.backend, *op), relative);
|
|
}
|
|
}
|
|
|
|
// Check for missing features
|
|
for backend in &backends {
|
|
if !supported_by.contains(backend) {
|
|
missing_features.push(ParityGap {
|
|
feature: format!("{:?}", op),
|
|
missing_in: *backend,
|
|
present_in: *supported_by.first().unwrap_or(&Backend::Nccl),
|
|
severity: "high".to_string(),
|
|
});
|
|
}
|
|
}
|
|
|
|
operation_parity.insert(
|
|
*op,
|
|
OperationParity {
|
|
operation: *op,
|
|
supported_all: supported_by.len() == backends.len(),
|
|
supported_by,
|
|
numerically_equivalent: true, // Would need actual verification
|
|
max_relative_error: 0.0,
|
|
},
|
|
);
|
|
}
|
|
|
|
// Calculate overall ranking
|
|
let mut backend_scores: HashMap<Backend, f64> = HashMap::new();
|
|
for ((backend, _), score) in &relative_performance {
|
|
*backend_scores.entry(*backend).or_insert(0.0) += score;
|
|
}
|
|
|
|
let mut overall_ranking: Vec<(Backend, f64)> = backend_scores.into_iter().collect();
|
|
overall_ranking.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap());
|
|
|
|
let supported_ops = operation_parity
|
|
.values()
|
|
.filter(|p| p.supported_all)
|
|
.count();
|
|
let overall_parity = (supported_ops as f64 / operation_parity.len() as f64) * 100.0;
|
|
|
|
ParityReport {
|
|
backends,
|
|
operation_parity,
|
|
overall_parity_percent: overall_parity,
|
|
missing_features,
|
|
performance_comparison: PerformanceComparison {
|
|
relative_performance,
|
|
fastest_for,
|
|
overall_ranking,
|
|
},
|
|
}
|
|
}
|
|
|
|
/// Export report to JSON
|
|
pub fn export_json(&self) -> Result<String> {
|
|
let report = self.generate_report();
|
|
serde_json::to_string_pretty(&report).map_err(|e| {
|
|
DistributedError::communication("benchmark", format!("JSON export failed: {}", e))
|
|
})
|
|
}
|
|
|
|
/// Generate a human-readable summary
|
|
pub fn summary_string(&self) -> String {
|
|
let report = self.generate_report();
|
|
let mut output = String::new();
|
|
|
|
output.push_str("=".repeat(70).as_str());
|
|
output.push_str("\nBackend Benchmark Report\n");
|
|
output.push_str("=".repeat(70).as_str());
|
|
output.push('\n');
|
|
|
|
// Summary
|
|
output.push_str("\nSummary:\n");
|
|
output.push_str(&format!(
|
|
" Total benchmarks: {}\n",
|
|
report.summary.total_benchmarks
|
|
));
|
|
output.push_str(&format!(" Successful: {}\n", report.summary.successful));
|
|
output.push_str(&format!(" Failed: {}\n", report.summary.failed));
|
|
output.push_str(&format!(
|
|
" Duration: {:.2}s\n",
|
|
report.summary.total_duration_secs
|
|
));
|
|
output.push_str(&format!(
|
|
" Best bandwidth: {:.2} GB/s ({})\n",
|
|
report.summary.best_bandwidth_gbps, report.summary.best_bandwidth_details
|
|
));
|
|
|
|
// Results by backend
|
|
output.push_str(&format!("\n{}\n", "-".repeat(70)));
|
|
output.push_str("Results by Backend:\n");
|
|
output.push_str(&format!("{}\n", "-".repeat(70)));
|
|
|
|
for (backend, results) in &report.by_backend {
|
|
output.push_str(&format!("\n{:?}:\n", backend));
|
|
for result in results.iter().filter(|r| r.success) {
|
|
output.push_str(&format!(
|
|
" {:?} {:>10} bytes: {:.2} us (p99: {:.2} us), {:.2} GB/s\n",
|
|
result.operation,
|
|
result.message_size,
|
|
result.latency.mean_us,
|
|
result.latency.p99_us,
|
|
result.bandwidth_gbps
|
|
));
|
|
}
|
|
}
|
|
|
|
// Parity
|
|
if let Some(parity) = &report.parity {
|
|
output.push_str(&format!("\n{}\n", "-".repeat(70)));
|
|
output.push_str("Feature Parity:\n");
|
|
output.push_str(&format!("{}\n", "-".repeat(70)));
|
|
output.push_str(&format!(
|
|
" Overall parity: {:.1}%\n",
|
|
parity.overall_parity_percent
|
|
));
|
|
|
|
if !parity.missing_features.is_empty() {
|
|
output.push_str("\n Missing features:\n");
|
|
for gap in &parity.missing_features {
|
|
output.push_str(&format!(
|
|
" - {} missing in {:?} (present in {:?})\n",
|
|
gap.feature, gap.missing_in, gap.present_in
|
|
));
|
|
}
|
|
}
|
|
|
|
output.push_str("\n Performance ranking:\n");
|
|
for (i, (backend, score)) in parity
|
|
.performance_comparison
|
|
.overall_ranking
|
|
.iter()
|
|
.enumerate()
|
|
{
|
|
output.push_str(&format!(" {}. {:?}: {:.2}\n", i + 1, backend, score));
|
|
}
|
|
}
|
|
|
|
output.push_str(&format!("\n{}\n", "=".repeat(70)));
|
|
output
|
|
}
|
|
}
|
|
|
|
// =============================================================================
|
|
// Utility Functions
|
|
// =============================================================================
|
|
|
|
/// Format bytes as human-readable string
|
|
pub fn format_bytes(bytes: usize) -> String {
|
|
const KB: usize = 1024;
|
|
const MB: usize = KB * 1024;
|
|
const GB: usize = MB * 1024;
|
|
|
|
if bytes >= GB {
|
|
format!("{:.2} GB", bytes as f64 / GB as f64)
|
|
} else if bytes >= MB {
|
|
format!("{:.2} MB", bytes as f64 / MB as f64)
|
|
} else if bytes >= KB {
|
|
format!("{:.2} KB", bytes as f64 / KB as f64)
|
|
} else {
|
|
format!("{} B", bytes)
|
|
}
|
|
}
|
|
|
|
/// Shared benchmark handle
|
|
pub type SharedBackendBenchmark = Arc<RwLock<BackendBenchmark>>;
|
|
|
|
/// Create a shared benchmark instance
|
|
pub fn shared_benchmark(config: BenchmarkConfig) -> SharedBackendBenchmark {
|
|
Arc::new(RwLock::new(BackendBenchmark::new(config)))
|
|
}
|
|
|
|
// =============================================================================
|
|
// Tests
|
|
// =============================================================================
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn test_benchmark_config_default() {
|
|
let config = BenchmarkConfig::default();
|
|
assert!(!config.message_sizes.is_empty());
|
|
assert!(!config.operations.is_empty());
|
|
assert!(config.iterations > 0);
|
|
}
|
|
|
|
#[test]
|
|
fn test_benchmark_config_quick() {
|
|
let config = BenchmarkConfig::quick();
|
|
assert!(config.iterations < BenchmarkConfig::default().iterations);
|
|
assert!(config.warmup_iterations < BenchmarkConfig::default().warmup_iterations);
|
|
}
|
|
|
|
#[test]
|
|
fn test_latency_stats() {
|
|
let latencies = vec![10.0, 20.0, 30.0, 40.0, 50.0];
|
|
let stats = LatencyStats::from_measurements(&latencies);
|
|
|
|
assert_eq!(stats.min_us, 10.0);
|
|
assert_eq!(stats.max_us, 50.0);
|
|
assert_eq!(stats.mean_us, 30.0);
|
|
assert_eq!(stats.median_us, 30.0);
|
|
}
|
|
|
|
#[test]
|
|
fn test_latency_stats_empty() {
|
|
let latencies: Vec<f64> = vec![];
|
|
let stats = LatencyStats::from_measurements(&latencies);
|
|
assert_eq!(stats.min_us, 0.0);
|
|
assert_eq!(stats.max_us, 0.0);
|
|
}
|
|
|
|
#[test]
|
|
fn test_format_bytes() {
|
|
assert_eq!(format_bytes(500), "500 B");
|
|
assert_eq!(format_bytes(1024), "1.00 KB");
|
|
assert_eq!(format_bytes(1024 * 1024), "1.00 MB");
|
|
assert_eq!(format_bytes(1024 * 1024 * 1024), "1.00 GB");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_benchmark_run() {
|
|
let config = BenchmarkConfig {
|
|
message_sizes: vec![1024],
|
|
operations: vec![CollectiveType::AllReduce],
|
|
iterations: 2,
|
|
warmup_iterations: 1,
|
|
backends: vec![Backend::Tcp],
|
|
num_gpus: 2,
|
|
verify_results: false,
|
|
timeout_secs: 60,
|
|
};
|
|
|
|
let mut benchmark = BackendBenchmark::new(config);
|
|
let results = benchmark.run().await.unwrap();
|
|
|
|
assert_eq!(results.len(), 1);
|
|
assert!(results[0].success);
|
|
}
|
|
|
|
#[test]
|
|
fn test_generate_report() {
|
|
let config = BenchmarkConfig {
|
|
message_sizes: vec![1024],
|
|
operations: vec![CollectiveType::AllReduce],
|
|
iterations: 2,
|
|
warmup_iterations: 1,
|
|
backends: vec![Backend::Tcp],
|
|
num_gpus: 2,
|
|
verify_results: false,
|
|
timeout_secs: 60,
|
|
};
|
|
|
|
let benchmark = BackendBenchmark::new(config);
|
|
|
|
// Add a result manually
|
|
benchmark.results.write().push(BenchmarkResult {
|
|
backend: Backend::Tcp,
|
|
operation: CollectiveType::AllReduce,
|
|
message_size: 1024,
|
|
latency: LatencyStats::from_measurements(&[100.0, 110.0, 105.0]),
|
|
bandwidth_gbps: 0.01,
|
|
iterations: 3,
|
|
success: true,
|
|
error: None,
|
|
});
|
|
|
|
let report = benchmark.generate_report();
|
|
assert_eq!(report.summary.total_benchmarks, 1);
|
|
assert_eq!(report.summary.successful, 1);
|
|
}
|
|
|
|
#[test]
|
|
fn test_parity_report() {
|
|
let config = BenchmarkConfig {
|
|
message_sizes: vec![1024],
|
|
operations: vec![CollectiveType::AllReduce],
|
|
iterations: 2,
|
|
warmup_iterations: 1,
|
|
backends: vec![Backend::Nccl, Backend::Rccl],
|
|
num_gpus: 2,
|
|
verify_results: false,
|
|
timeout_secs: 60,
|
|
};
|
|
|
|
let benchmark = BackendBenchmark::new(config);
|
|
|
|
// Add results for both backends
|
|
benchmark.results.write().push(BenchmarkResult {
|
|
backend: Backend::Nccl,
|
|
operation: CollectiveType::AllReduce,
|
|
message_size: 1024,
|
|
latency: LatencyStats::from_measurements(&[100.0]),
|
|
bandwidth_gbps: 0.01,
|
|
iterations: 1,
|
|
success: true,
|
|
error: None,
|
|
});
|
|
|
|
benchmark.results.write().push(BenchmarkResult {
|
|
backend: Backend::Rccl,
|
|
operation: CollectiveType::AllReduce,
|
|
message_size: 1024,
|
|
latency: LatencyStats::from_measurements(&[110.0]),
|
|
bandwidth_gbps: 0.009,
|
|
iterations: 1,
|
|
success: true,
|
|
error: None,
|
|
});
|
|
|
|
let parity = benchmark.verify_parity();
|
|
assert_eq!(parity.backends.len(), 2);
|
|
assert_eq!(parity.overall_parity_percent, 100.0);
|
|
}
|
|
}
|