374 lines
12 KiB
Rust
374 lines
12 KiB
Rust
//! # Streaming Performance Benchmarks
|
|
//!
|
|
//! Comprehensive benchmarks to validate sub-millisecond latency and
|
|
//! high-throughput streaming requirements.
|
|
|
|
use criterion::{BenchmarkId, Criterion, black_box, criterion_group, criterion_main};
|
|
use rtx_streaming::{StreamingConfig, StreamingServer};
|
|
use std::time::{Duration, Instant};
|
|
use tokio::runtime::Runtime;
|
|
|
|
/// Benchmark sub-millisecond latency requirement
|
|
fn bench_sub_millisecond_latency(c: &mut Criterion) {
|
|
let rt = Runtime::new().unwrap();
|
|
|
|
let mut group = c.benchmark_group("latency");
|
|
group.measurement_time(Duration::from_secs(10));
|
|
group.sample_size(1000);
|
|
|
|
// Setup streaming server
|
|
let server = rt.block_on(async {
|
|
let config = StreamingConfig {
|
|
target_latency: Duration::from_micros(500), // 0.5ms target
|
|
..Default::default()
|
|
};
|
|
StreamingServer::new(config).await.unwrap()
|
|
});
|
|
|
|
group.bench_function("single_token_generation", |b| {
|
|
b.to_async(&rt).iter(|| async {
|
|
let start = Instant::now();
|
|
let _result = server.stream_inference(black_box("Hello world")).await;
|
|
let latency = start.elapsed();
|
|
|
|
// Assert sub-millisecond latency
|
|
assert!(
|
|
latency < Duration::from_millis(1),
|
|
"Latency {} exceeds 1ms requirement",
|
|
latency.as_micros()
|
|
);
|
|
|
|
latency
|
|
});
|
|
});
|
|
|
|
// Test different prompt lengths
|
|
for prompt_length in [10, 50, 100, 500].iter() {
|
|
let prompt = "test ".repeat(*prompt_length);
|
|
group.bench_with_input(
|
|
BenchmarkId::new("prompt_length", prompt_length),
|
|
prompt_length,
|
|
|b, _| {
|
|
b.to_async(&rt).iter(|| async {
|
|
let start = Instant::now();
|
|
let _result = server.stream_inference(black_box(&prompt)).await;
|
|
let latency = start.elapsed();
|
|
|
|
// Latency should remain sub-millisecond regardless of prompt length
|
|
assert!(latency < Duration::from_millis(1));
|
|
|
|
latency
|
|
});
|
|
},
|
|
);
|
|
}
|
|
|
|
group.finish();
|
|
}
|
|
|
|
/// Benchmark high throughput (>1000 QPS) requirement
|
|
fn bench_high_throughput(c: &mut Criterion) {
|
|
let rt = Runtime::new().unwrap();
|
|
|
|
let mut group = c.benchmark_group("throughput");
|
|
group.measurement_time(Duration::from_secs(30));
|
|
group.sample_size(100);
|
|
|
|
let server = rt.block_on(async {
|
|
let config = StreamingConfig {
|
|
max_connections: 2000,
|
|
..Default::default()
|
|
};
|
|
StreamingServer::new(config).await.unwrap()
|
|
});
|
|
|
|
// Test concurrent streaming connections
|
|
for concurrent_connections in [100, 500, 1000, 1500].iter() {
|
|
group.bench_with_input(
|
|
BenchmarkId::new("concurrent_connections", concurrent_connections),
|
|
concurrent_connections,
|
|
|b, &conn_count| {
|
|
b.to_async(&rt).iter(|| async {
|
|
let start = Instant::now();
|
|
|
|
let mut handles = Vec::new();
|
|
for i in 0..conn_count {
|
|
let server_clone = server.clone();
|
|
let handle = tokio::spawn(async move {
|
|
server_clone
|
|
.stream_inference(&format!("prompt_{}", i))
|
|
.await
|
|
});
|
|
handles.push(handle);
|
|
}
|
|
|
|
// Wait for all to complete
|
|
for handle in handles {
|
|
handle.await.unwrap().unwrap();
|
|
}
|
|
|
|
let elapsed = start.elapsed();
|
|
let qps = conn_count as f64 / elapsed.as_secs_f64();
|
|
|
|
// Assert >1000 QPS requirement
|
|
if conn_count >= 1000 {
|
|
assert!(
|
|
qps > 1000.0,
|
|
"Throughput {} QPS is below 1000 QPS requirement",
|
|
qps
|
|
);
|
|
}
|
|
|
|
qps
|
|
});
|
|
},
|
|
);
|
|
}
|
|
|
|
group.finish();
|
|
}
|
|
|
|
/// Benchmark memory efficiency (<10% overhead per connection)
|
|
fn bench_memory_efficiency(c: &mut Criterion) {
|
|
let rt = Runtime::new().unwrap();
|
|
|
|
let mut group = c.benchmark_group("memory_efficiency");
|
|
group.measurement_time(Duration::from_secs(15));
|
|
group.sample_size(50);
|
|
|
|
let server = rt.block_on(async {
|
|
let config = StreamingConfig::default();
|
|
StreamingServer::new(config).await.unwrap()
|
|
});
|
|
|
|
// Measure memory overhead per connection
|
|
for connection_count in [10, 50, 100, 500].iter() {
|
|
group.bench_with_input(
|
|
BenchmarkId::new("connection_count", connection_count),
|
|
connection_count,
|
|
|b, &conn_count| {
|
|
b.to_async(&rt).iter(|| async {
|
|
// Measure baseline memory
|
|
let baseline_memory = get_memory_usage();
|
|
|
|
// Create connections
|
|
let mut connections = Vec::new();
|
|
for i in 0..conn_count {
|
|
let connection = server
|
|
.create_connection(&format!("client_{}", i))
|
|
.await
|
|
.unwrap();
|
|
connections.push(connection);
|
|
}
|
|
|
|
let streaming_memory = get_memory_usage();
|
|
let overhead_per_connection = (streaming_memory - baseline_memory) / conn_count;
|
|
let overhead_percentage =
|
|
(overhead_per_connection as f64 / baseline_memory as f64) * 100.0;
|
|
|
|
// Assert <10% overhead per connection
|
|
assert!(
|
|
overhead_percentage < 10.0,
|
|
"Memory overhead {}% exceeds 10% requirement",
|
|
overhead_percentage
|
|
);
|
|
|
|
overhead_percentage
|
|
});
|
|
},
|
|
);
|
|
}
|
|
|
|
group.finish();
|
|
}
|
|
|
|
/// Benchmark connection stability (zero drops)
|
|
fn bench_connection_stability(c: &mut Criterion) {
|
|
let rt = Runtime::new().unwrap();
|
|
|
|
let mut group = c.benchmark_group("connection_stability");
|
|
group.measurement_time(Duration::from_secs(20));
|
|
group.sample_size(100);
|
|
|
|
let server = rt.block_on(async {
|
|
let config = StreamingConfig::default();
|
|
StreamingServer::new(config).await.unwrap()
|
|
});
|
|
|
|
group.bench_function("connection_stability_test", |b| {
|
|
b.to_async(&rt).iter(|| async {
|
|
let mut connection_drops = 0;
|
|
let total_connections = black_box(1000);
|
|
|
|
for i in 0..total_connections {
|
|
match server
|
|
.create_connection(&format!("stable_client_{}", i))
|
|
.await
|
|
{
|
|
Ok(_) => {
|
|
// Simulate some streaming activity
|
|
tokio::time::sleep(Duration::from_micros(100)).await;
|
|
}
|
|
Err(_) => {
|
|
connection_drops += 1;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Assert zero connection drops
|
|
assert_eq!(
|
|
connection_drops, 0,
|
|
"Had {} connection drops, expected 0",
|
|
connection_drops
|
|
);
|
|
|
|
connection_drops
|
|
});
|
|
});
|
|
|
|
group.finish();
|
|
}
|
|
|
|
/// Benchmark backpressure handling
|
|
fn bench_backpressure_handling(c: &mut Criterion) {
|
|
let rt = Runtime::new().unwrap();
|
|
|
|
let mut group = c.benchmark_group("backpressure");
|
|
group.measurement_time(Duration::from_secs(10));
|
|
group.sample_size(50);
|
|
|
|
let server = rt.block_on(async {
|
|
let config = StreamingConfig {
|
|
backpressure_threshold: 0.7,
|
|
..Default::default()
|
|
};
|
|
StreamingServer::new(config).await.unwrap()
|
|
});
|
|
|
|
group.bench_function("overload_scenario", |b| {
|
|
b.to_async(&rt).iter(|| async {
|
|
// Create overload scenario
|
|
let result = server.handle_overload_scenario().await;
|
|
|
|
// Assert graceful handling
|
|
assert!(
|
|
result.is_ok(),
|
|
"Backpressure handling failed: {:?}",
|
|
result.err()
|
|
);
|
|
|
|
let handled_gracefully = result.unwrap();
|
|
assert!(
|
|
handled_gracefully,
|
|
"System did not handle overload gracefully"
|
|
);
|
|
|
|
handled_gracefully
|
|
});
|
|
});
|
|
|
|
group.finish();
|
|
}
|
|
|
|
/// Benchmark end-to-end streaming performance
|
|
fn bench_end_to_end_streaming(c: &mut Criterion) {
|
|
let rt = Runtime::new().unwrap();
|
|
|
|
let mut group = c.benchmark_group("end_to_end");
|
|
group.measurement_time(Duration::from_secs(30));
|
|
group.sample_size(200);
|
|
|
|
let server = rt.block_on(async {
|
|
let config = StreamingConfig {
|
|
target_latency: Duration::from_micros(800),
|
|
max_connections: 1500,
|
|
..Default::default()
|
|
};
|
|
StreamingServer::new(config).await.unwrap()
|
|
});
|
|
|
|
group.bench_function("realistic_workload", |b| {
|
|
b.to_async(&rt).iter(|| async {
|
|
let start = Instant::now();
|
|
|
|
// Simulate realistic mixed workload
|
|
let mut handles = Vec::new();
|
|
|
|
// 70% normal requests
|
|
for i in 0..700 {
|
|
let server_clone = server.clone();
|
|
let handle = tokio::spawn(async move {
|
|
server_clone
|
|
.stream_inference(&format!("normal_prompt_{}", i))
|
|
.await
|
|
});
|
|
handles.push(handle);
|
|
}
|
|
|
|
// 20% concurrent batch requests
|
|
for i in 0..200 {
|
|
let server_clone = server.clone();
|
|
let handle = tokio::spawn(async move {
|
|
server_clone
|
|
.stream_inference(&format!("batch_prompt_{}", i))
|
|
.await
|
|
});
|
|
handles.push(handle);
|
|
}
|
|
|
|
// 10% high-priority requests
|
|
for i in 0..100 {
|
|
let server_clone = server.clone();
|
|
let handle = tokio::spawn(async move {
|
|
server_clone
|
|
.stream_inference(&format!("priority_prompt_{}", i))
|
|
.await
|
|
});
|
|
handles.push(handle);
|
|
}
|
|
|
|
// Wait for all to complete
|
|
for handle in handles {
|
|
handle.await.unwrap().unwrap();
|
|
}
|
|
|
|
let elapsed = start.elapsed();
|
|
let total_requests = 1000;
|
|
let qps = total_requests as f64 / elapsed.as_secs_f64();
|
|
|
|
// Validate performance requirements
|
|
assert!(qps > 1000.0, "End-to-end QPS {} below requirement", qps);
|
|
|
|
let avg_latency = elapsed / total_requests;
|
|
assert!(
|
|
avg_latency < Duration::from_millis(1),
|
|
"Average latency {} exceeds requirement",
|
|
avg_latency.as_micros()
|
|
);
|
|
|
|
(qps, avg_latency)
|
|
});
|
|
});
|
|
|
|
group.finish();
|
|
}
|
|
|
|
/// Mock helper function for memory measurement
|
|
fn get_memory_usage() -> usize {
|
|
// In real implementation, this would use system APIs
|
|
// For benchmarking, return a realistic mock value
|
|
1024 * 1024 * 50 // 50MB baseline
|
|
}
|
|
|
|
criterion_group!(
|
|
streaming_benches,
|
|
bench_sub_millisecond_latency,
|
|
bench_high_throughput,
|
|
bench_memory_efficiency,
|
|
bench_connection_stability,
|
|
bench_backpressure_handling,
|
|
bench_end_to_end_streaming
|
|
);
|
|
|
|
criterion_main!(streaming_benches);
|