262 lines
7.6 KiB
Rust
262 lines
7.6 KiB
Rust
/*!
|
|
End-to-End Latency Benchmarks
|
|
|
|
Comprehensive benchmarks measuring end-to-end latency across different
|
|
components and scenarios. These benchmarks establish performance baselines
|
|
and detect performance regressions.
|
|
|
|
## Benchmark Categories
|
|
|
|
1. **Inference Latency**: Single request inference latency
|
|
2. **Batch Processing**: Batched inference latency optimization
|
|
3. **Model Loading**: Cold start and warm start latency
|
|
4. **Pipeline Latency**: Complete ML pipeline execution time
|
|
5. **Cross-Component**: Latency across component boundaries
|
|
6. **Network Latency**: API and network overhead measurement
|
|
|
|
## Usage
|
|
|
|
```bash
|
|
# Run all latency benchmarks
|
|
cargo bench --bench end_to_end_latency
|
|
|
|
# Run specific benchmark group
|
|
cargo bench --bench end_to_end_latency -- inference
|
|
|
|
# Generate HTML report
|
|
cargo bench --bench end_to_end_latency -- --output-format html
|
|
```
|
|
*/
|
|
|
|
use criterion::{black_box, criterion_group, criterion_main, Criterion, BenchmarkId};
|
|
use rustytorch_integration_tests::*;
|
|
use std::time::Duration;
|
|
use tokio::runtime::Runtime;
|
|
|
|
fn benchmark_inference_latency(c: &mut Criterion) {
|
|
let rt = Runtime::new().unwrap();
|
|
|
|
let mut group = c.benchmark_group("inference_latency");
|
|
group.measurement_time(Duration::from_secs(10));
|
|
group.sample_size(100);
|
|
|
|
// Test different input sizes
|
|
let input_sizes = vec![128, 512, 1024, 2048];
|
|
|
|
for size in input_sizes {
|
|
group.bench_with_input(
|
|
BenchmarkId::new("single_request", size),
|
|
&size,
|
|
|b, &size| {
|
|
b.to_async(&rt).iter(|| async {
|
|
let input = vec![0.5f32; size];
|
|
// Mock inference call
|
|
benchmark_single_inference(black_box(input)).await
|
|
});
|
|
},
|
|
);
|
|
}
|
|
|
|
group.finish();
|
|
}
|
|
|
|
fn benchmark_batch_processing(c: &mut Criterion) {
|
|
let rt = Runtime::new().unwrap();
|
|
|
|
let mut group = c.benchmark_group("batch_processing");
|
|
group.measurement_time(Duration::from_secs(15));
|
|
|
|
let batch_sizes = vec![1, 4, 8, 16, 32];
|
|
let input_size = 512;
|
|
|
|
for batch_size in batch_sizes {
|
|
group.bench_with_input(
|
|
BenchmarkId::new("batch_inference", batch_size),
|
|
&batch_size,
|
|
|b, &batch_size| {
|
|
b.to_async(&rt).iter(|| async {
|
|
let batch = vec![vec![0.5f32; input_size]; batch_size];
|
|
benchmark_batch_inference(black_box(batch)).await
|
|
});
|
|
},
|
|
);
|
|
}
|
|
|
|
group.finish();
|
|
}
|
|
|
|
fn benchmark_model_loading(c: &mut Criterion) {
|
|
let rt = Runtime::new().unwrap();
|
|
|
|
let mut group = c.benchmark_group("model_loading");
|
|
group.measurement_time(Duration::from_secs(20));
|
|
group.sample_size(10); // Fewer samples for expensive operations
|
|
|
|
group.bench_function("cold_start", |b| {
|
|
b.to_async(&rt).iter(|| async {
|
|
benchmark_cold_start().await
|
|
});
|
|
});
|
|
|
|
group.bench_function("warm_start", |b| {
|
|
b.to_async(&rt).iter(|| async {
|
|
benchmark_warm_start().await
|
|
});
|
|
});
|
|
|
|
group.finish();
|
|
}
|
|
|
|
fn benchmark_pipeline_latency(c: &mut Criterion) {
|
|
let rt = Runtime::new().unwrap();
|
|
|
|
let mut group = c.benchmark_group("pipeline_latency");
|
|
group.measurement_time(Duration::from_secs(30));
|
|
group.sample_size(20);
|
|
|
|
let pipeline_types = vec!["classification", "generation", "embedding"];
|
|
|
|
for pipeline_type in pipeline_types {
|
|
group.bench_with_input(
|
|
BenchmarkId::new("end_to_end", pipeline_type),
|
|
&pipeline_type,
|
|
|b, &pipeline_type| {
|
|
b.to_async(&rt).iter(|| async {
|
|
benchmark_pipeline_execution(black_box(pipeline_type)).await
|
|
});
|
|
},
|
|
);
|
|
}
|
|
|
|
group.finish();
|
|
}
|
|
|
|
fn benchmark_cross_component_latency(c: &mut Criterion) {
|
|
let rt = Runtime::new().unwrap();
|
|
|
|
let mut group = c.benchmark_group("cross_component");
|
|
group.measurement_time(Duration::from_secs(15));
|
|
|
|
group.bench_function("tensor_to_autograd", |b| {
|
|
b.to_async(&rt).iter(|| async {
|
|
benchmark_tensor_autograd_boundary().await
|
|
});
|
|
});
|
|
|
|
group.bench_function("autograd_to_transformers", |b| {
|
|
b.to_async(&rt).iter(|| async {
|
|
benchmark_autograd_transformers_boundary().await
|
|
});
|
|
});
|
|
|
|
group.bench_function("transformers_to_serving", |b| {
|
|
b.to_async(&rt).iter(|| async {
|
|
benchmark_transformers_serving_boundary().await
|
|
});
|
|
});
|
|
|
|
group.finish();
|
|
}
|
|
|
|
fn benchmark_network_latency(c: &mut Criterion) {
|
|
let rt = Runtime::new().unwrap();
|
|
|
|
let mut group = c.benchmark_group("network_latency");
|
|
group.measurement_time(Duration::from_secs(10));
|
|
|
|
group.bench_function("api_overhead", |b| {
|
|
b.to_async(&rt).iter(|| async {
|
|
benchmark_api_overhead().await
|
|
});
|
|
});
|
|
|
|
group.bench_function("serialization", |b| {
|
|
b.to_async(&rt).iter(|| async {
|
|
let data = vec![0.5f32; 1024];
|
|
benchmark_serialization_overhead(black_box(data)).await
|
|
});
|
|
});
|
|
|
|
group.finish();
|
|
}
|
|
|
|
// Mock benchmark functions - would be replaced with actual implementations
|
|
async fn benchmark_single_inference(input: Vec<f32>) -> Vec<f32> {
|
|
// Simulate inference computation
|
|
tokio::time::sleep(Duration::from_micros(100)).await;
|
|
vec![0.1, 0.2, 0.3] // Mock output
|
|
}
|
|
|
|
async fn benchmark_batch_inference(batch: Vec<Vec<f32>>) -> Vec<Vec<f32>> {
|
|
// Simulate batch inference
|
|
let batch_size = batch.len();
|
|
tokio::time::sleep(Duration::from_micros(50 + batch_size as u64 * 20)).await;
|
|
vec![vec![0.1, 0.2, 0.3]; batch_size]
|
|
}
|
|
|
|
async fn benchmark_cold_start() -> String {
|
|
// Simulate model loading from disk
|
|
tokio::time::sleep(Duration::from_millis(500)).await;
|
|
"model_loaded".to_string()
|
|
}
|
|
|
|
async fn benchmark_warm_start() -> String {
|
|
// Simulate model already in memory
|
|
tokio::time::sleep(Duration::from_millis(50)).await;
|
|
"model_ready".to_string()
|
|
}
|
|
|
|
async fn benchmark_pipeline_execution(pipeline_type: &str) -> String {
|
|
// Simulate complete pipeline execution
|
|
let base_latency = match pipeline_type {
|
|
"classification" => 200,
|
|
"generation" => 1000,
|
|
"embedding" => 100,
|
|
_ => 300,
|
|
};
|
|
|
|
tokio::time::sleep(Duration::from_millis(base_latency)).await;
|
|
format!("{}_result", pipeline_type)
|
|
}
|
|
|
|
async fn benchmark_tensor_autograd_boundary() -> String {
|
|
// Simulate tensor to autograd operation
|
|
tokio::time::sleep(Duration::from_micros(50)).await;
|
|
"gradient_computed".to_string()
|
|
}
|
|
|
|
async fn benchmark_autograd_transformers_boundary() -> String {
|
|
// Simulate autograd to transformers operation
|
|
tokio::time::sleep(Duration::from_micros(75)).await;
|
|
"forward_pass_complete".to_string()
|
|
}
|
|
|
|
async fn benchmark_transformers_serving_boundary() -> String {
|
|
// Simulate transformers to serving operation
|
|
tokio::time::sleep(Duration::from_micros(25)).await;
|
|
"response_serialized".to_string()
|
|
}
|
|
|
|
async fn benchmark_api_overhead() -> String {
|
|
// Simulate API call overhead
|
|
tokio::time::sleep(Duration::from_micros(10)).await;
|
|
"api_response".to_string()
|
|
}
|
|
|
|
async fn benchmark_serialization_overhead(data: Vec<f32>) -> Vec<u8> {
|
|
// Simulate serialization overhead
|
|
tokio::time::sleep(Duration::from_micros(5 + data.len() as u64 / 100)).await;
|
|
vec![0; data.len() * 4] // Mock serialized data
|
|
}
|
|
|
|
criterion_group!(
|
|
benches,
|
|
benchmark_inference_latency,
|
|
benchmark_batch_processing,
|
|
benchmark_model_loading,
|
|
benchmark_pipeline_latency,
|
|
benchmark_cross_component_latency,
|
|
benchmark_network_latency
|
|
);
|
|
|
|
criterion_main!(benches); |