rustytorch/docs/book/src/performance/profiling.md

Profiling

Find performance bottlenecks in RustyTorch++ applications.

Built-in Profiler

use rtx_bench::Profiler;

let profiler = Profiler::new();

profiler.scope("train_step", || {
    profiler.scope("forward", || {
        model.forward(&input)
    })?;

    profiler.scope("backward", || {
        loss.backward()
    })?;

    profiler.scope("optimizer", || {
        optimizer.step()
    })
})?;

profiler.print_summary();

Output:

┌─────────────┬──────────┬────────┬─────────┐
│ Scope       │ Time(ms) │ Calls  │ % Total │
├─────────────┼──────────┼────────┼─────────┤
│ train_step  │   45.23  │   100  │  100.0% │
│ ├─forward   │   12.34  │   100  │   27.3% │
│ ├─backward  │   28.56  │   100  │   63.1% │
│ └─optimizer │    4.33  │   100  │    9.6% │
└─────────────┴──────────┴────────┴─────────┘

NVIDIA Nsight Systems

Installation

# Download from NVIDIA
# https://developer.nvidia.com/nsight-systems

# Or via package manager
apt install nsight-systems

Profile Application

# Basic profile
nsys profile --stats=true ./target/release/my-app

# With GPU trace
nsys profile -t cuda,nvtx --stats=true ./target/release/my-app

# Export for GUI analysis
nsys profile -o report.nsys-rep ./target/release/my-app

NVTX Annotations

use rtx_runtime::profiling::nvtx;

// Mark regions
nvtx::range_push("Forward Pass");
let output = model.forward(&input)?;
nvtx::range_pop();

nvtx::range_push("Backward Pass");
output.backward()?;
nvtx::range_pop();

// Name CUDA streams
nvtx::name_stream(&stream, "compute");

NVIDIA Nsight Compute

For detailed kernel analysis:

# Profile specific kernel
ncu --set full --target-processes all ./target/release/my-app

# Roofline analysis
ncu --set roofline ./target/release/my-app

Memory Profiling

Track Allocations

use rtx_memory::AllocationTracker;

AllocationTracker::enable();

// Run code...
train_epoch(&model, &data)?;

let report = AllocationTracker::report();
println!("Peak memory: {} MB", report.peak_mb);
println!("Total allocated: {} MB", report.total_allocated_mb);
println!("Allocation count: {}", report.allocation_count);

// Find largest allocations
for alloc in report.top_allocations(10) {
    println!("{}: {} MB at {}", alloc.name, alloc.size_mb, alloc.location);
}

Memory Timeline

use rtx_memory::MemoryTimeline;

let timeline = MemoryTimeline::record(|| {
    train_epoch(&model, &data)
})?;

// Export for visualization
timeline.export_json("memory_timeline.json")?;

CPU Profiling

With perf

# Record CPU profile
perf record -g ./target/release/my-app

# Generate flamegraph
perf script | stackcollapse-perf.pl | flamegraph.pl > flame.svg

With cargo-flamegraph

cargo install flamegraph

# Generate flamegraph
cargo flamegraph --release --bin my-app

Distributed Profiling

Multi-GPU Profile

use rtx_distributed::Profiler as DistProfiler;

let profiler = DistProfiler::new(world_size)?;

profiler.scope("all_reduce", || {
    dist.all_reduce(&gradients)
})?;

// Aggregate across ranks
let report = profiler.collect_all()?;
if rank == 0 {
    report.print_summary();
}

Common Bottlenecks

CPU-GPU Synchronization

Symptom: Long gaps between GPU kernels in timeline
Cause: Implicit synchronization

Fix:
- Use async operations
- Avoid .item() calls in loops
- Batch CPU operations

Memory Bandwidth

Symptom: Low SM utilization, high memory traffic
Cause: Large tensors, poor access patterns

Fix:
- Use mixed precision (FP16)
- Optimize memory layout
- Use Flash Attention

Kernel Launch Overhead

Symptom: Many small kernels, low GPU utilization
Cause: Too many individual operations

Fix:
- Use CUDA graphs
- Kernel fusion
- Batch operations

Performance Dashboard

use rtx_monitoring::Dashboard;

let dashboard = Dashboard::new()
    .with_throughput()
    .with_latency()
    .with_memory()
    .with_gpu_utilization();

// Start monitoring server
dashboard.serve(9090).await?;

// Access at http://localhost:9090

Automated Analysis

use rtx_bench::PerformanceAnalyzer;

let analyzer = PerformanceAnalyzer::new(&device)?;

let analysis = analyzer.analyze(|| {
    model.forward(&input)
})?;

// Get recommendations
for recommendation in analysis.recommendations() {
    println!("{}: {}", recommendation.category, recommendation.message);
}

Next Steps

• GPU Optimization - Apply optimizations
• Memory Management - Fix memory issues
• Troubleshooting - Common issues