Files
rustytorch/scripts/run_benchmarks.sh
T
2026-03-04 00:08:42 +00:00

537 lines
19 KiB
Bash
Executable File

#!/bin/bash
# RustyTorch++ Benchmark Runner with Auto-Report Generation
# Usage: ./scripts/run_benchmarks.sh [--pinn-only|--all] [--output-dir DIR]
set -e
# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
CYAN='\033[0;36m'
NC='\033[0m' # No Color
# Default configuration
BENCHMARK_SCOPE="pinn"
OUTPUT_DIR="benchmark_reports"
TIMESTAMP=$(date +"%Y%m%d_%H%M%S")
PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
# Auto-detect platform and GPU backend
if [[ "$(uname)" == "Darwin" ]]; then
GPU_BACKEND="metal"
PLATFORM="macOS"
else
GPU_BACKEND="cuda"
PLATFORM="Linux"
fi
# Parse arguments
while [[ $# -gt 0 ]]; do
case $1 in
--pinn-only)
BENCHMARK_SCOPE="pinn"
shift
;;
--all)
BENCHMARK_SCOPE="all"
shift
;;
--output-dir)
OUTPUT_DIR="$2"
shift 2
;;
-h|--help)
echo "RustyTorch++ Benchmark Runner"
echo ""
echo "Usage: $0 [OPTIONS]"
echo ""
echo "Options:"
echo " --pinn-only Run only PINN Helmholtz benchmarks (default)"
echo " --all Run all benchmarks in the workspace"
echo " --output-dir Output directory for reports (default: benchmark_reports)"
echo " -h, --help Show this help message"
exit 0
;;
*)
echo "Unknown option: $1"
exit 1
;;
esac
done
# Create output directory
mkdir -p "$PROJECT_ROOT/$OUTPUT_DIR"
echo -e "${CYAN}╔════════════════════════════════════════════════════════════╗${NC}"
echo -e "${CYAN}║ RustyTorch++ Benchmark Suite ║${NC}"
echo -e "${CYAN}╚════════════════════════════════════════════════════════════╝${NC}"
echo ""
echo -e "Platform: ${YELLOW}$PLATFORM${NC} | GPU Backend: ${YELLOW}$GPU_BACKEND${NC}"
echo ""
# =============================================================================
# Phase 1: Collect System Information
# =============================================================================
echo -e "${BLUE}[1/4]${NC} Collecting system information..."
SYSTEM_INFO_FILE="$PROJECT_ROOT/$OUTPUT_DIR/system_info_${TIMESTAMP}.txt"
RAW_SYSTEM_JSON="$PROJECT_ROOT/$OUTPUT_DIR/system_${TIMESTAMP}.json"
# Collect OS info based on platform
if [[ "$PLATFORM" == "macOS" ]]; then
OS_NAME="macOS"
OS_VERSION=$(sw_vers -productVersion 2>/dev/null || echo "Unknown")
KERNEL=$(uname -r)
# Collect CPU info (macOS)
CPU_MODEL=$(sysctl -n machdep.cpu.brand_string 2>/dev/null || echo "Apple Silicon")
CPU_CORES=$(sysctl -n hw.physicalcpu 2>/dev/null || echo "Unknown")
CPU_THREADS=$(sysctl -n hw.logicalcpu 2>/dev/null || echo "Unknown")
CPU_SOCKETS="1"
CPU_MAX_MHZ="N/A"
# Collect GPU info (Metal/Apple Silicon)
GPU_INFO=$(system_profiler SPDisplaysDataType 2>/dev/null | grep -A2 "Chipset Model" || echo "")
if [[ -n "$GPU_INFO" ]]; then
GPU_NAME=$(echo "$GPU_INFO" | grep "Chipset Model" | cut -d':' -f2 | xargs || echo "Apple GPU")
GPU_MEMORY=$(system_profiler SPDisplaysDataType 2>/dev/null | grep "VRAM" | cut -d':' -f2 | xargs || echo "Unified Memory")
GPU_DRIVER="Metal"
GPU_COMPUTE="Apple GPU"
else
GPU_NAME="Apple Silicon GPU"
GPU_MEMORY="Unified Memory"
GPU_DRIVER="Metal"
GPU_COMPUTE="Apple GPU"
fi
else
# Linux
OS_NAME=$(cat /etc/os-release 2>/dev/null | grep "^NAME=" | cut -d'"' -f2 || echo "Unknown")
OS_VERSION=$(cat /etc/os-release 2>/dev/null | grep "^VERSION=" | cut -d'"' -f2 || echo "Unknown")
KERNEL=$(uname -r)
# Collect CPU info (Linux)
CPU_MODEL=$(lscpu 2>/dev/null | grep "Model name" | cut -d':' -f2 | xargs || echo "Unknown")
CPU_CORES=$(lscpu 2>/dev/null | grep "^CPU(s):" | cut -d':' -f2 | xargs || echo "Unknown")
CPU_THREADS=$(lscpu 2>/dev/null | grep "Thread(s) per core" | cut -d':' -f2 | xargs || echo "1")
CPU_SOCKETS=$(lscpu 2>/dev/null | grep "Socket(s)" | cut -d':' -f2 | xargs || echo "1")
CPU_MAX_MHZ=$(lscpu 2>/dev/null | grep "CPU max MHz" | cut -d':' -f2 | xargs || echo "Unknown")
# Collect GPU info (NVIDIA/CUDA)
if command -v nvidia-smi &> /dev/null; then
GPU_INFO=$(nvidia-smi --query-gpu=name,memory.total,driver_version,compute_cap --format=csv,noheader 2>/dev/null || echo "")
if [[ -n "$GPU_INFO" ]]; then
GPU_NAME=$(echo "$GPU_INFO" | cut -d',' -f1 | xargs)
GPU_MEMORY=$(echo "$GPU_INFO" | cut -d',' -f2 | xargs)
GPU_DRIVER=$(echo "$GPU_INFO" | cut -d',' -f3 | xargs)
GPU_COMPUTE=$(echo "$GPU_INFO" | cut -d',' -f4 | xargs)
else
GPU_NAME="Not detected"
GPU_MEMORY="N/A"
GPU_DRIVER="N/A"
GPU_COMPUTE="N/A"
fi
else
GPU_NAME="nvidia-smi not available"
GPU_MEMORY="N/A"
GPU_DRIVER="N/A"
GPU_COMPUTE="N/A"
fi
fi
# Collect memory info
if [[ "$PLATFORM" == "macOS" ]]; then
TOTAL_MEM_BYTES=$(sysctl -n hw.memsize 2>/dev/null || echo "0")
TOTAL_MEM="$((TOTAL_MEM_BYTES / 1024 / 1024 / 1024)) GB"
else
TOTAL_MEM=$(free -h 2>/dev/null | grep "Mem:" | awk '{print $2}' || echo "Unknown")
fi
# Get git commit
GIT_COMMIT=$(cd "$PROJECT_ROOT" && git rev-parse --short HEAD 2>/dev/null || echo "unknown")
GIT_BRANCH=$(cd "$PROJECT_ROOT" && git branch --show-current 2>/dev/null || echo "unknown")
# Write system info text file
cat > "$SYSTEM_INFO_FILE" << EOF
RustyTorch++ Benchmark System Information
=========================================
Generated: $(date)
Commit: $GIT_COMMIT ($GIT_BRANCH)
OPERATING SYSTEM
----------------
OS: $OS_NAME $OS_VERSION
Kernel: $KERNEL
Architecture: $(uname -m)
CPU
---
Model: $CPU_MODEL
Cores: $CPU_CORES (Threads per core: $CPU_THREADS)
Sockets: $CPU_SOCKETS
Max Frequency: $CPU_MAX_MHZ MHz
GPU (NVIDIA)
------------
Model: $GPU_NAME
Memory: $GPU_MEMORY
Driver Version: $GPU_DRIVER
Compute Capability: $GPU_COMPUTE
SYSTEM MEMORY
-------------
Total: $TOTAL_MEM
EOF
# Write system info JSON
cat > "$RAW_SYSTEM_JSON" << EOF
{
"timestamp": "$(date -Iseconds)",
"commit": "$GIT_COMMIT",
"branch": "$GIT_BRANCH",
"os": {
"name": "$OS_NAME",
"version": "$OS_VERSION",
"kernel": "$KERNEL",
"arch": "$(uname -m)"
},
"cpu": {
"model": "$CPU_MODEL",
"cores": "$CPU_CORES",
"threads_per_core": "$CPU_THREADS",
"sockets": "$CPU_SOCKETS",
"max_mhz": "$CPU_MAX_MHZ"
},
"gpu": {
"name": "$GPU_NAME",
"memory": "$GPU_MEMORY",
"driver": "$GPU_DRIVER",
"compute_capability": "$GPU_COMPUTE"
},
"memory": {
"total": "$TOTAL_MEM"
}
}
EOF
echo -e " ${GREEN}${NC} System info saved to $SYSTEM_INFO_FILE"
# =============================================================================
# Phase 2: Run Benchmarks
# =============================================================================
echo -e "${BLUE}[2/4]${NC} Running benchmarks (scope: $BENCHMARK_SCOPE, backend: $GPU_BACKEND)..."
BENCHMARK_OUTPUT="$PROJECT_ROOT/$OUTPUT_DIR/benchmark_output_${TIMESTAMP}.txt"
BENCHMARK_RAW="$PROJECT_ROOT/$OUTPUT_DIR/benchmark_raw_${TIMESTAMP}.json"
cd "$PROJECT_ROOT"
if [[ "$BENCHMARK_SCOPE" == "pinn" ]]; then
echo " Running PINN Helmholtz benchmarks with --features $GPU_BACKEND..."
(cd examples/pinn_mre_helmholtz && cargo bench --features $GPU_BACKEND 2>&1) | tee "$BENCHMARK_OUTPUT"
else
echo " Running all workspace benchmarks with --features $GPU_BACKEND..."
cargo bench --workspace --features $GPU_BACKEND 2>&1 | tee "$BENCHMARK_OUTPUT"
fi
echo -e " ${GREEN}${NC} Benchmark output saved to $BENCHMARK_OUTPUT"
# =============================================================================
# Phase 3: Parse Benchmark Results
# =============================================================================
echo -e "${BLUE}[3/4]${NC} Parsing benchmark results..."
# Extract benchmark results from Criterion output
# Format: benchmark_name followed by "time: [low mean high]"
PARSED_RESULTS=$(grep -A2 "^forward_pass\|^training_step\|^training_100" "$BENCHMARK_OUTPUT" 2>/dev/null | grep -v "^--$" || echo "")
# Count benchmarks
BENCHMARK_COUNT=$(echo "$PARSED_RESULTS" | grep -c "time:" 2>/dev/null || echo "0")
echo -e " ${GREEN}${NC} Parsed $BENCHMARK_COUNT benchmark results"
# =============================================================================
# Phase 4: Generate Reports
# =============================================================================
echo -e "${BLUE}[4/4]${NC} Generating reports..."
HTML_REPORT="$PROJECT_ROOT/$OUTPUT_DIR/benchmark_report_${TIMESTAMP}.html"
MD_REPORT="$PROJECT_ROOT/$OUTPUT_DIR/benchmark_report_${TIMESTAMP}.md"
# Generate HTML Report
cat > "$HTML_REPORT" << 'HTMLEOF'
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>RustyTorch++ Benchmark Report</title>
<style>
:root {
--primary: #4f46e5;
--primary-dark: #3730a3;
--secondary: #06b6d4;
--success: #10b981;
--warning: #f59e0b;
--danger: #ef4444;
--bg-dark: #1e1e2e;
--bg-card: #2a2a3e;
--text: #e2e8f0;
--text-muted: #94a3b8;
--border: #3f3f5a;
}
* { margin: 0; padding: 0; box-sizing: border-box; }
body {
font-family: 'Segoe UI', system-ui, -apple-system, sans-serif;
background: linear-gradient(135deg, var(--bg-dark) 0%, #0f0f1a 100%);
color: var(--text);
line-height: 1.6;
min-height: 100vh;
}
.container { max-width: 1200px; margin: 0 auto; padding: 2rem; }
header {
text-align: center;
padding: 3rem 0;
border-bottom: 1px solid var(--border);
margin-bottom: 2rem;
}
h1 {
font-size: 2.5rem;
background: linear-gradient(135deg, var(--primary), var(--secondary));
-webkit-background-clip: text;
-webkit-text-fill-color: transparent;
background-clip: text;
margin-bottom: 0.5rem;
}
.subtitle { color: var(--text-muted); font-size: 1.1rem; }
.timestamp { margin-top: 1rem; color: var(--text-muted); font-size: 0.9rem; }
section { margin-bottom: 3rem; }
h2 {
font-size: 1.5rem;
margin-bottom: 1.5rem;
padding-bottom: 0.5rem;
border-bottom: 2px solid var(--primary);
display: flex;
align-items: center;
gap: 0.5rem;
}
h2::before {
content: '';
display: inline-block;
width: 8px;
height: 8px;
background: var(--primary);
border-radius: 50%;
}
.card {
background: var(--bg-card);
border-radius: 12px;
padding: 1.5rem;
margin-bottom: 1rem;
border: 1px solid var(--border);
box-shadow: 0 4px 6px rgba(0, 0, 0, 0.3);
}
.card-title {
font-size: 1.1rem;
font-weight: 600;
margin-bottom: 1rem;
color: var(--secondary);
}
.specs-grid {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(280px, 1fr));
gap: 1rem;
}
.spec-item {
display: flex;
align-items: center;
gap: 1rem;
padding: 1rem;
background: rgba(79, 70, 229, 0.1);
border-radius: 8px;
border-left: 3px solid var(--primary);
}
.spec-icon { font-size: 1.5rem; width: 40px; text-align: center; }
.spec-label {
color: var(--text-muted);
font-size: 0.85rem;
text-transform: uppercase;
letter-spacing: 0.05em;
}
.spec-value { font-weight: 600; color: var(--text); }
table { width: 100%; border-collapse: collapse; margin-top: 1rem; }
th, td {
padding: 0.75rem 1rem;
text-align: left;
border-bottom: 1px solid var(--border);
}
th {
background: rgba(79, 70, 229, 0.2);
font-weight: 600;
color: var(--secondary);
text-transform: uppercase;
font-size: 0.85rem;
}
tr:hover { background: rgba(79, 70, 229, 0.1); }
.metric-highlight { color: var(--success); font-weight: 700; }
pre {
background: rgba(0, 0, 0, 0.3);
padding: 1rem;
border-radius: 8px;
overflow-x: auto;
font-size: 0.85rem;
}
footer {
text-align: center;
padding: 2rem;
border-top: 1px solid var(--border);
color: var(--text-muted);
}
</style>
</head>
<body>
<div class="container">
<header>
<h1>RustyTorch++ Benchmark Report</h1>
<p class="subtitle">Production-Ready GPU-Accelerated ML Framework in Pure Rust</p>
<p class="timestamp">Generated: TIMESTAMP_PLACEHOLDER | Commit: COMMIT_PLACEHOLDER</p>
</header>
<section id="system-info">
<h2>System Specifications</h2>
<div class="specs-grid">
<div class="spec-item">
<div class="spec-icon">🖥️</div>
<div>
<div class="spec-label">Operating System</div>
<div class="spec-value">OS_PLACEHOLDER</div>
</div>
</div>
<div class="spec-item">
<div class="spec-icon">⚙️</div>
<div>
<div class="spec-label">Kernel</div>
<div class="spec-value">KERNEL_PLACEHOLDER</div>
</div>
</div>
<div class="spec-item">
<div class="spec-icon">🔲</div>
<div>
<div class="spec-label">CPU</div>
<div class="spec-value">CPU_PLACEHOLDER</div>
</div>
</div>
<div class="spec-item">
<div class="spec-icon">🎮</div>
<div>
<div class="spec-label">GPU</div>
<div class="spec-value">GPU_PLACEHOLDER</div>
</div>
</div>
<div class="spec-item">
<div class="spec-icon">💾</div>
<div>
<div class="spec-label">GPU Memory</div>
<div class="spec-value">GPU_MEM_PLACEHOLDER</div>
</div>
</div>
<div class="spec-item">
<div class="spec-icon">🧠</div>
<div>
<div class="spec-label">System Memory</div>
<div class="spec-value">MEM_PLACEHOLDER</div>
</div>
</div>
</div>
</section>
<section id="benchmarks">
<h2>Benchmark Results</h2>
<div class="card">
<div class="card-title">Raw Criterion Output</div>
<pre>BENCHMARK_OUTPUT_PLACEHOLDER</pre>
</div>
</section>
<footer>
<p>RustyTorch++ | Production-Ready GPU-Accelerated ML Framework in Pure Rust</p>
<p style="margin-top: 0.5rem;">Auto-generated by run_benchmarks.sh</p>
</footer>
</div>
</body>
</html>
HTMLEOF
# Replace placeholders in HTML
sed -i "s|TIMESTAMP_PLACEHOLDER|$(date)|g" "$HTML_REPORT"
sed -i "s|COMMIT_PLACEHOLDER|$GIT_COMMIT|g" "$HTML_REPORT"
sed -i "s|OS_PLACEHOLDER|$OS_NAME $OS_VERSION|g" "$HTML_REPORT"
sed -i "s|KERNEL_PLACEHOLDER|$KERNEL|g" "$HTML_REPORT"
sed -i "s|CPU_PLACEHOLDER|$CPU_MODEL|g" "$HTML_REPORT"
sed -i "s|GPU_PLACEHOLDER|$GPU_NAME|g" "$HTML_REPORT"
sed -i "s|GPU_MEM_PLACEHOLDER|$GPU_MEMORY|g" "$HTML_REPORT"
sed -i "s|MEM_PLACEHOLDER|$TOTAL_MEM|g" "$HTML_REPORT"
# Extract and insert benchmark results (escape for sed)
BENCHMARK_SUMMARY=$(echo "$PARSED_RESULTS" | head -60 | sed 's/&/\&amp;/g; s/</\&lt;/g; s/>/\&gt;/g' | tr '\n' '§' | sed 's/§/<br>/g')
sed -i "s|BENCHMARK_OUTPUT_PLACEHOLDER|$BENCHMARK_SUMMARY|g" "$HTML_REPORT"
echo -e " ${GREEN}${NC} HTML report: $HTML_REPORT"
# Generate Markdown Report
cat > "$MD_REPORT" << EOF
# RustyTorch++ Benchmark Report
**Generated:** $(date)
**Commit:** $GIT_COMMIT ($GIT_BRANCH)
---
## System Specifications
| Component | Details |
|-----------|---------|
| **OS** | $OS_NAME $OS_VERSION |
| **Kernel** | $KERNEL |
| **CPU** | $CPU_MODEL |
| **CPU Config** | $CPU_CORES cores @ $CPU_MAX_MHZ MHz max |
| **GPU** | $GPU_NAME |
| **GPU Memory** | $GPU_MEMORY |
| **Compute Capability** | $GPU_COMPUTE |
| **Driver** | $GPU_DRIVER |
| **System Memory** | $TOTAL_MEM |
---
## Benchmark Results
\`\`\`
$PARSED_RESULTS
\`\`\`
---
*Auto-generated by run_benchmarks.sh*
EOF
echo -e " ${GREEN}${NC} Markdown report: $MD_REPORT"
# Create symlinks to latest reports
ln -sf "benchmark_report_${TIMESTAMP}.html" "$PROJECT_ROOT/$OUTPUT_DIR/latest_report.html"
ln -sf "benchmark_report_${TIMESTAMP}.md" "$PROJECT_ROOT/$OUTPUT_DIR/latest_report.md"
ln -sf "system_info_${TIMESTAMP}.txt" "$PROJECT_ROOT/$OUTPUT_DIR/latest_system_info.txt"
echo ""
echo -e "${GREEN}╔════════════════════════════════════════════════════════════╗${NC}"
echo -e "${GREEN}║ Benchmark Complete! ║${NC}"
echo -e "${GREEN}╚════════════════════════════════════════════════════════════╝${NC}"
echo ""
echo -e "Reports generated in: ${CYAN}$OUTPUT_DIR/${NC}"
echo -e " - HTML Report: ${CYAN}latest_report.html${NC}"
echo -e " - Markdown: ${CYAN}latest_report.md${NC}"
echo -e " - System Info: ${CYAN}latest_system_info.txt${NC}"
echo -e " - Raw Output: ${CYAN}benchmark_output_${TIMESTAMP}.txt${NC}"
echo ""
echo -e "View HTML report: ${YELLOW}firefox $OUTPUT_DIR/latest_report.html${NC}"