#!/bin/bash # RustyTorch++ Benchmark Runner with Auto-Report Generation # Usage: ./scripts/run_benchmarks.sh [--pinn-only|--all] [--output-dir DIR] set -e # Colors for output RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' CYAN='\033[0;36m' NC='\033[0m' # No Color # Default configuration BENCHMARK_SCOPE="pinn" OUTPUT_DIR="benchmark_reports" TIMESTAMP=$(date +"%Y%m%d_%H%M%S") PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" # Auto-detect platform and GPU backend if [[ "$(uname)" == "Darwin" ]]; then GPU_BACKEND="metal" PLATFORM="macOS" else GPU_BACKEND="cuda" PLATFORM="Linux" fi # Parse arguments while [[ $# -gt 0 ]]; do case $1 in --pinn-only) BENCHMARK_SCOPE="pinn" shift ;; --all) BENCHMARK_SCOPE="all" shift ;; --output-dir) OUTPUT_DIR="$2" shift 2 ;; -h|--help) echo "RustyTorch++ Benchmark Runner" echo "" echo "Usage: $0 [OPTIONS]" echo "" echo "Options:" echo " --pinn-only Run only PINN Helmholtz benchmarks (default)" echo " --all Run all benchmarks in the workspace" echo " --output-dir Output directory for reports (default: benchmark_reports)" echo " -h, --help Show this help message" exit 0 ;; *) echo "Unknown option: $1" exit 1 ;; esac done # Create output directory mkdir -p "$PROJECT_ROOT/$OUTPUT_DIR" echo -e "${CYAN}╔════════════════════════════════════════════════════════════╗${NC}" echo -e "${CYAN}║ RustyTorch++ Benchmark Suite ║${NC}" echo -e "${CYAN}╚════════════════════════════════════════════════════════════╝${NC}" echo "" echo -e "Platform: ${YELLOW}$PLATFORM${NC} | GPU Backend: ${YELLOW}$GPU_BACKEND${NC}" echo "" # ============================================================================= # Phase 1: Collect System Information # ============================================================================= echo -e "${BLUE}[1/4]${NC} Collecting system information..." SYSTEM_INFO_FILE="$PROJECT_ROOT/$OUTPUT_DIR/system_info_${TIMESTAMP}.txt" RAW_SYSTEM_JSON="$PROJECT_ROOT/$OUTPUT_DIR/system_${TIMESTAMP}.json" # Collect OS info based on platform if [[ "$PLATFORM" == "macOS" ]]; then OS_NAME="macOS" OS_VERSION=$(sw_vers -productVersion 2>/dev/null || echo "Unknown") KERNEL=$(uname -r) # Collect CPU info (macOS) CPU_MODEL=$(sysctl -n machdep.cpu.brand_string 2>/dev/null || echo "Apple Silicon") CPU_CORES=$(sysctl -n hw.physicalcpu 2>/dev/null || echo "Unknown") CPU_THREADS=$(sysctl -n hw.logicalcpu 2>/dev/null || echo "Unknown") CPU_SOCKETS="1" CPU_MAX_MHZ="N/A" # Collect GPU info (Metal/Apple Silicon) GPU_INFO=$(system_profiler SPDisplaysDataType 2>/dev/null | grep -A2 "Chipset Model" || echo "") if [[ -n "$GPU_INFO" ]]; then GPU_NAME=$(echo "$GPU_INFO" | grep "Chipset Model" | cut -d':' -f2 | xargs || echo "Apple GPU") GPU_MEMORY=$(system_profiler SPDisplaysDataType 2>/dev/null | grep "VRAM" | cut -d':' -f2 | xargs || echo "Unified Memory") GPU_DRIVER="Metal" GPU_COMPUTE="Apple GPU" else GPU_NAME="Apple Silicon GPU" GPU_MEMORY="Unified Memory" GPU_DRIVER="Metal" GPU_COMPUTE="Apple GPU" fi else # Linux OS_NAME=$(cat /etc/os-release 2>/dev/null | grep "^NAME=" | cut -d'"' -f2 || echo "Unknown") OS_VERSION=$(cat /etc/os-release 2>/dev/null | grep "^VERSION=" | cut -d'"' -f2 || echo "Unknown") KERNEL=$(uname -r) # Collect CPU info (Linux) CPU_MODEL=$(lscpu 2>/dev/null | grep "Model name" | cut -d':' -f2 | xargs || echo "Unknown") CPU_CORES=$(lscpu 2>/dev/null | grep "^CPU(s):" | cut -d':' -f2 | xargs || echo "Unknown") CPU_THREADS=$(lscpu 2>/dev/null | grep "Thread(s) per core" | cut -d':' -f2 | xargs || echo "1") CPU_SOCKETS=$(lscpu 2>/dev/null | grep "Socket(s)" | cut -d':' -f2 | xargs || echo "1") CPU_MAX_MHZ=$(lscpu 2>/dev/null | grep "CPU max MHz" | cut -d':' -f2 | xargs || echo "Unknown") # Collect GPU info (NVIDIA/CUDA) if command -v nvidia-smi &> /dev/null; then GPU_INFO=$(nvidia-smi --query-gpu=name,memory.total,driver_version,compute_cap --format=csv,noheader 2>/dev/null || echo "") if [[ -n "$GPU_INFO" ]]; then GPU_NAME=$(echo "$GPU_INFO" | cut -d',' -f1 | xargs) GPU_MEMORY=$(echo "$GPU_INFO" | cut -d',' -f2 | xargs) GPU_DRIVER=$(echo "$GPU_INFO" | cut -d',' -f3 | xargs) GPU_COMPUTE=$(echo "$GPU_INFO" | cut -d',' -f4 | xargs) else GPU_NAME="Not detected" GPU_MEMORY="N/A" GPU_DRIVER="N/A" GPU_COMPUTE="N/A" fi else GPU_NAME="nvidia-smi not available" GPU_MEMORY="N/A" GPU_DRIVER="N/A" GPU_COMPUTE="N/A" fi fi # Collect memory info if [[ "$PLATFORM" == "macOS" ]]; then TOTAL_MEM_BYTES=$(sysctl -n hw.memsize 2>/dev/null || echo "0") TOTAL_MEM="$((TOTAL_MEM_BYTES / 1024 / 1024 / 1024)) GB" else TOTAL_MEM=$(free -h 2>/dev/null | grep "Mem:" | awk '{print $2}' || echo "Unknown") fi # Get git commit GIT_COMMIT=$(cd "$PROJECT_ROOT" && git rev-parse --short HEAD 2>/dev/null || echo "unknown") GIT_BRANCH=$(cd "$PROJECT_ROOT" && git branch --show-current 2>/dev/null || echo "unknown") # Write system info text file cat > "$SYSTEM_INFO_FILE" << EOF RustyTorch++ Benchmark System Information ========================================= Generated: $(date) Commit: $GIT_COMMIT ($GIT_BRANCH) OPERATING SYSTEM ---------------- OS: $OS_NAME $OS_VERSION Kernel: $KERNEL Architecture: $(uname -m) CPU --- Model: $CPU_MODEL Cores: $CPU_CORES (Threads per core: $CPU_THREADS) Sockets: $CPU_SOCKETS Max Frequency: $CPU_MAX_MHZ MHz GPU (NVIDIA) ------------ Model: $GPU_NAME Memory: $GPU_MEMORY Driver Version: $GPU_DRIVER Compute Capability: $GPU_COMPUTE SYSTEM MEMORY ------------- Total: $TOTAL_MEM EOF # Write system info JSON cat > "$RAW_SYSTEM_JSON" << EOF { "timestamp": "$(date -Iseconds)", "commit": "$GIT_COMMIT", "branch": "$GIT_BRANCH", "os": { "name": "$OS_NAME", "version": "$OS_VERSION", "kernel": "$KERNEL", "arch": "$(uname -m)" }, "cpu": { "model": "$CPU_MODEL", "cores": "$CPU_CORES", "threads_per_core": "$CPU_THREADS", "sockets": "$CPU_SOCKETS", "max_mhz": "$CPU_MAX_MHZ" }, "gpu": { "name": "$GPU_NAME", "memory": "$GPU_MEMORY", "driver": "$GPU_DRIVER", "compute_capability": "$GPU_COMPUTE" }, "memory": { "total": "$TOTAL_MEM" } } EOF echo -e " ${GREEN}✓${NC} System info saved to $SYSTEM_INFO_FILE" # ============================================================================= # Phase 2: Run Benchmarks # ============================================================================= echo -e "${BLUE}[2/4]${NC} Running benchmarks (scope: $BENCHMARK_SCOPE, backend: $GPU_BACKEND)..." BENCHMARK_OUTPUT="$PROJECT_ROOT/$OUTPUT_DIR/benchmark_output_${TIMESTAMP}.txt" BENCHMARK_RAW="$PROJECT_ROOT/$OUTPUT_DIR/benchmark_raw_${TIMESTAMP}.json" cd "$PROJECT_ROOT" if [[ "$BENCHMARK_SCOPE" == "pinn" ]]; then echo " Running PINN Helmholtz benchmarks with --features $GPU_BACKEND..." (cd examples/pinn_mre_helmholtz && cargo bench --features $GPU_BACKEND 2>&1) | tee "$BENCHMARK_OUTPUT" else echo " Running all workspace benchmarks with --features $GPU_BACKEND..." cargo bench --workspace --features $GPU_BACKEND 2>&1 | tee "$BENCHMARK_OUTPUT" fi echo -e " ${GREEN}✓${NC} Benchmark output saved to $BENCHMARK_OUTPUT" # ============================================================================= # Phase 3: Parse Benchmark Results # ============================================================================= echo -e "${BLUE}[3/4]${NC} Parsing benchmark results..." # Extract benchmark results from Criterion output # Format: benchmark_name followed by "time: [low mean high]" PARSED_RESULTS=$(grep -A2 "^forward_pass\|^training_step\|^training_100" "$BENCHMARK_OUTPUT" 2>/dev/null | grep -v "^--$" || echo "") # Count benchmarks BENCHMARK_COUNT=$(echo "$PARSED_RESULTS" | grep -c "time:" 2>/dev/null || echo "0") echo -e " ${GREEN}✓${NC} Parsed $BENCHMARK_COUNT benchmark results" # ============================================================================= # Phase 4: Generate Reports # ============================================================================= echo -e "${BLUE}[4/4]${NC} Generating reports..." HTML_REPORT="$PROJECT_ROOT/$OUTPUT_DIR/benchmark_report_${TIMESTAMP}.html" MD_REPORT="$PROJECT_ROOT/$OUTPUT_DIR/benchmark_report_${TIMESTAMP}.md" # Generate HTML Report cat > "$HTML_REPORT" << 'HTMLEOF' RustyTorch++ Benchmark Report

RustyTorch++ Benchmark Report

Production-Ready GPU-Accelerated ML Framework in Pure Rust

Generated: TIMESTAMP_PLACEHOLDER | Commit: COMMIT_PLACEHOLDER

System Specifications

🖥️
Operating System
OS_PLACEHOLDER
⚙️
Kernel
KERNEL_PLACEHOLDER
🔲
CPU
CPU_PLACEHOLDER
🎮
GPU
GPU_PLACEHOLDER
💾
GPU Memory
GPU_MEM_PLACEHOLDER
🧠
System Memory
MEM_PLACEHOLDER

Benchmark Results

Raw Criterion Output
BENCHMARK_OUTPUT_PLACEHOLDER
HTMLEOF # Replace placeholders in HTML sed -i "s|TIMESTAMP_PLACEHOLDER|$(date)|g" "$HTML_REPORT" sed -i "s|COMMIT_PLACEHOLDER|$GIT_COMMIT|g" "$HTML_REPORT" sed -i "s|OS_PLACEHOLDER|$OS_NAME $OS_VERSION|g" "$HTML_REPORT" sed -i "s|KERNEL_PLACEHOLDER|$KERNEL|g" "$HTML_REPORT" sed -i "s|CPU_PLACEHOLDER|$CPU_MODEL|g" "$HTML_REPORT" sed -i "s|GPU_PLACEHOLDER|$GPU_NAME|g" "$HTML_REPORT" sed -i "s|GPU_MEM_PLACEHOLDER|$GPU_MEMORY|g" "$HTML_REPORT" sed -i "s|MEM_PLACEHOLDER|$TOTAL_MEM|g" "$HTML_REPORT" # Extract and insert benchmark results (escape for sed) BENCHMARK_SUMMARY=$(echo "$PARSED_RESULTS" | head -60 | sed 's/&/\&/g; s//\>/g' | tr '\n' '§' | sed 's/§/
/g') sed -i "s|BENCHMARK_OUTPUT_PLACEHOLDER|$BENCHMARK_SUMMARY|g" "$HTML_REPORT" echo -e " ${GREEN}✓${NC} HTML report: $HTML_REPORT" # Generate Markdown Report cat > "$MD_REPORT" << EOF # RustyTorch++ Benchmark Report **Generated:** $(date) **Commit:** $GIT_COMMIT ($GIT_BRANCH) --- ## System Specifications | Component | Details | |-----------|---------| | **OS** | $OS_NAME $OS_VERSION | | **Kernel** | $KERNEL | | **CPU** | $CPU_MODEL | | **CPU Config** | $CPU_CORES cores @ $CPU_MAX_MHZ MHz max | | **GPU** | $GPU_NAME | | **GPU Memory** | $GPU_MEMORY | | **Compute Capability** | $GPU_COMPUTE | | **Driver** | $GPU_DRIVER | | **System Memory** | $TOTAL_MEM | --- ## Benchmark Results \`\`\` $PARSED_RESULTS \`\`\` --- *Auto-generated by run_benchmarks.sh* EOF echo -e " ${GREEN}✓${NC} Markdown report: $MD_REPORT" # Create symlinks to latest reports ln -sf "benchmark_report_${TIMESTAMP}.html" "$PROJECT_ROOT/$OUTPUT_DIR/latest_report.html" ln -sf "benchmark_report_${TIMESTAMP}.md" "$PROJECT_ROOT/$OUTPUT_DIR/latest_report.md" ln -sf "system_info_${TIMESTAMP}.txt" "$PROJECT_ROOT/$OUTPUT_DIR/latest_system_info.txt" echo "" echo -e "${GREEN}╔════════════════════════════════════════════════════════════╗${NC}" echo -e "${GREEN}║ Benchmark Complete! ║${NC}" echo -e "${GREEN}╚════════════════════════════════════════════════════════════╝${NC}" echo "" echo -e "Reports generated in: ${CYAN}$OUTPUT_DIR/${NC}" echo -e " - HTML Report: ${CYAN}latest_report.html${NC}" echo -e " - Markdown: ${CYAN}latest_report.md${NC}" echo -e " - System Info: ${CYAN}latest_system_info.txt${NC}" echo -e " - Raw Output: ${CYAN}benchmark_output_${TIMESTAMP}.txt${NC}" echo "" echo -e "View HTML report: ${YELLOW}firefox $OUTPUT_DIR/latest_report.html${NC}"