#!/usr/bin/env python3 """ Generate HTML comparison report for PyTorch vs RustyTorch++ PINN benchmarks. This script parses benchmark results from: - PyTorch JSON output (from benchmark_runner.py) - Rust Criterion text output And generates: - comparison_report.html (visual report with charts) - comparison_report.md (markdown summary) Usage: python generate_comparison_report.py \ --output-dir ./benchmark_reports/hostname-rust-py-pinn-benchmark-12-10-2025/ \ --system-info system_info.json \ --pytorch-cpu pytorch_cpu_results.json \ --pytorch-gpu pytorch_gpu_results.json \ --rust-cpu rust_cpu_output.txt \ --rust-gpu rust_gpu_output.txt """ import argparse import json import re from pathlib import Path from datetime import datetime from typing import Dict, List, Optional, Tuple from dataclasses import dataclass @dataclass class BenchmarkResult: """Parsed benchmark result.""" name: str n_points: int mean_time_us: float # microseconds std_time_us: float = 0.0 framework: str = "" device: str = "" def parse_pytorch_json(filepath: Path) -> List[BenchmarkResult]: """Parse PyTorch benchmark JSON output.""" results = [] if not filepath.exists(): return results with open(filepath) as f: data = json.load(f) framework = data.get("framework", "pytorch") device = data.get("device", "unknown") for bench in data.get("benchmarks", []): name = bench.get("name", "unknown") n_points = bench.get("n_points", 0) mean_ms = bench.get("mean_time_ms", 0) std_ms = bench.get("std_time_ms", 0) results.append(BenchmarkResult( name=name, n_points=n_points, mean_time_us=mean_ms * 1000, # ms -> us std_time_us=std_ms * 1000, framework=framework, device=device.split(":")[0] if ":" in device else device )) return results def parse_criterion_output(filepath: Path) -> List[BenchmarkResult]: """Parse Rust Criterion text output.""" results = [] if not filepath.exists(): return results content = filepath.read_text() # Pattern: benchmark_name/n_points time: [low mean high] # Example: forward_pass/200_points time: [40.123 us 41.456 us 42.789 us] pattern = r'(\S+)/(\d+)_points\s+time:\s+\[[\d.]+ [nuµm]s\s+([\d.]+)\s+([nuµm]s)' for match in re.finditer(pattern, content): name, n_points, mean_val, unit = match.groups() n_points = int(n_points) mean_val = float(mean_val) # Convert to microseconds if unit == "ns": mean_us = mean_val / 1000 elif unit in ("us", "µs"): mean_us = mean_val elif unit == "ms": mean_us = mean_val * 1000 else: mean_us = mean_val results.append(BenchmarkResult( name=name, n_points=n_points, mean_time_us=mean_us, framework="rustytorch", device="gpu" if "cuda" in str(filepath).lower() else "cpu" )) return results def load_system_info(filepath: Path) -> Dict: """Load system info JSON.""" if not filepath.exists(): return {} with open(filepath) as f: return json.load(f) def calculate_speedup(pytorch_us: float, rust_us: float) -> Tuple[float, str]: """Calculate speedup ratio and winner.""" if pytorch_us <= 0 or rust_us <= 0: return 1.0, "tie" if rust_us < pytorch_us: speedup = pytorch_us / rust_us winner = "rust" else: speedup = rust_us / pytorch_us winner = "pytorch" return speedup, winner def generate_html_report( output_dir: Path, system_info: Dict, pytorch_cpu: List[BenchmarkResult], pytorch_gpu: List[BenchmarkResult], rust_cpu: List[BenchmarkResult], rust_gpu: List[BenchmarkResult] ) -> None: """Generate HTML comparison report with charts.""" # Build comparison data benchmarks = {} # Organize by (name, n_points) for result in pytorch_cpu: key = (result.name, result.n_points) if key not in benchmarks: benchmarks[key] = {"pytorch_cpu": None, "pytorch_gpu": None, "rust_cpu": None, "rust_gpu": None} benchmarks[key]["pytorch_cpu"] = result.mean_time_us for result in pytorch_gpu: key = (result.name, result.n_points) if key not in benchmarks: benchmarks[key] = {"pytorch_cpu": None, "pytorch_gpu": None, "rust_cpu": None, "rust_gpu": None} benchmarks[key]["pytorch_gpu"] = result.mean_time_us for result in rust_cpu: key = (result.name, result.n_points) if key not in benchmarks: benchmarks[key] = {"pytorch_cpu": None, "pytorch_gpu": None, "rust_cpu": None, "rust_gpu": None} benchmarks[key]["rust_cpu"] = result.mean_time_us for result in rust_gpu: key = (result.name, result.n_points) if key not in benchmarks: benchmarks[key] = {"pytorch_cpu": None, "pytorch_gpu": None, "rust_cpu": None, "rust_gpu": None} benchmarks[key]["rust_gpu"] = result.mean_time_us # Build table rows table_rows = [] chart_data_cpu = [] chart_data_gpu = [] for (name, n_points), data in sorted(benchmarks.items()): py_cpu = data["pytorch_cpu"] py_gpu = data["pytorch_gpu"] rust_cpu_val = data["rust_cpu"] rust_gpu_val = data["rust_gpu"] # CPU speedup cpu_speedup = "" cpu_winner = "" if py_cpu and rust_cpu_val: speedup, winner = calculate_speedup(py_cpu, rust_cpu_val) cpu_speedup = f"{speedup:.2f}x" cpu_winner = winner chart_data_cpu.append({ "name": f"{name}/{n_points}", "pytorch": py_cpu, "rust": rust_cpu_val }) # GPU speedup gpu_speedup = "" gpu_winner = "" if py_gpu and rust_gpu_val: speedup, winner = calculate_speedup(py_gpu, rust_gpu_val) gpu_speedup = f"{speedup:.2f}x" gpu_winner = winner chart_data_gpu.append({ "name": f"{name}/{n_points}", "pytorch": py_gpu, "rust": rust_gpu_val }) def fmt(val): if val is None: return "-" if val >= 1000: return f"{val/1000:.2f} ms" return f"{val:.1f} µs" def winner_class(winner, framework): if winner == framework: return "winner" return "" table_rows.append(f""" {name} {n_points} {fmt(py_cpu)} {fmt(rust_cpu_val)} {cpu_speedup} {fmt(py_gpu)} {fmt(rust_gpu_val)} {gpu_speedup} """) table_html = "\n".join(table_rows) # System info sys_os = system_info.get("os", {}) sys_cpu = system_info.get("cpu", {}) sys_gpu = system_info.get("gpu", {}) sys_mem = system_info.get("memory", {}) hostname = system_info.get("hostname", "unknown") timestamp = system_info.get("timestamp", datetime.now().isoformat()) commit = system_info.get("commit", "unknown") html = f""" PyTorch vs RustyTorch++ PINN Benchmark

PyTorch vs RustyTorch++ PINN Benchmark

Physics-Informed Neural Network Performance Comparison

Host: {hostname} | Generated: {timestamp} | Commit: {commit}

System Specifications

💻
Operating System
{sys_os.get('name', 'Unknown')} {sys_os.get('version', '')}
⚙
CPU
{sys_cpu.get('model', 'Unknown')}
🎮
GPU
{sys_gpu.get('name', 'Unknown')}
💾
GPU Memory
{sys_gpu.get('memory', 'Unknown')}
🧠
System Memory
{sys_mem.get('total', 'Unknown')}
🔍
Compute Capability
{sys_gpu.get('compute_capability', 'N/A')}
PyTorch
RustyTorch++

Benchmark Results

{table_html}
Benchmark Points PyTorch CPU Rust CPU CPU Speedup PyTorch GPU Rust GPU GPU Speedup

Visual Comparison

CPU Performance (lower is better)
GPU Performance (lower is better)
""" html_path = output_dir / "comparison_report.html" html_path.write_text(html) print(f" HTML report: {html_path}") def generate_markdown_report( output_dir: Path, system_info: Dict, pytorch_cpu: List[BenchmarkResult], pytorch_gpu: List[BenchmarkResult], rust_cpu: List[BenchmarkResult], rust_gpu: List[BenchmarkResult] ) -> None: """Generate Markdown summary report.""" hostname = system_info.get("hostname", "unknown") timestamp = system_info.get("timestamp", datetime.now().isoformat()) commit = system_info.get("commit", "unknown") sys_os = system_info.get("os", {}) sys_cpu = system_info.get("cpu", {}) sys_gpu = system_info.get("gpu", {}) sys_mem = system_info.get("memory", {}) # Build comparison table benchmarks = {} for result in pytorch_cpu + pytorch_gpu + rust_cpu + rust_gpu: key = (result.name, result.n_points) if key not in benchmarks: benchmarks[key] = {"pytorch_cpu": None, "pytorch_gpu": None, "rust_cpu": None, "rust_gpu": None} if result.framework == "pytorch" and result.device == "cpu": benchmarks[key]["pytorch_cpu"] = result.mean_time_us elif result.framework == "pytorch" and "cuda" in result.device.lower(): benchmarks[key]["pytorch_gpu"] = result.mean_time_us elif result.framework == "rustytorch" and result.device == "cpu": benchmarks[key]["rust_cpu"] = result.mean_time_us elif result.framework == "rustytorch" and result.device == "gpu": benchmarks[key]["rust_gpu"] = result.mean_time_us table_rows = [] for (name, n_points), data in sorted(benchmarks.items()): py_cpu = data["pytorch_cpu"] py_gpu = data["pytorch_gpu"] rust_cpu_val = data["rust_cpu"] rust_gpu_val = data["rust_gpu"] def fmt(val): if val is None: return "-" if val >= 1000: return f"{val/1000:.2f} ms" return f"{val:.1f} µs" cpu_speedup = "" if py_cpu and rust_cpu_val: speedup, winner = calculate_speedup(py_cpu, rust_cpu_val) cpu_speedup = f"{speedup:.2f}x ({winner})" gpu_speedup = "" if py_gpu and rust_gpu_val: speedup, winner = calculate_speedup(py_gpu, rust_gpu_val) gpu_speedup = f"{speedup:.2f}x ({winner})" table_rows.append( f"| {name} | {n_points} | {fmt(py_cpu)} | {fmt(rust_cpu_val)} | {cpu_speedup} | {fmt(py_gpu)} | {fmt(rust_gpu_val)} | {gpu_speedup} |" ) table_str = "\n".join(table_rows) md = f"""# PyTorch vs RustyTorch++ PINN Benchmark **Host:** {hostname} **Generated:** {timestamp} **Commit:** {commit} --- ## System Specifications | Component | Details | |-----------|---------| | **OS** | {sys_os.get('name', 'Unknown')} {sys_os.get('version', '')} | | **CPU** | {sys_cpu.get('model', 'Unknown')} | | **GPU** | {sys_gpu.get('name', 'Unknown')} | | **GPU Memory** | {sys_gpu.get('memory', 'Unknown')} | | **System Memory** | {sys_mem.get('total', 'Unknown')} | --- ## Benchmark Results | Benchmark | Points | PyTorch CPU | Rust CPU | CPU Speedup | PyTorch GPU | Rust GPU | GPU Speedup | |-----------|--------|-------------|----------|-------------|-------------|----------|-------------| {table_str} --- *Auto-generated by run_pinn_comparison.sh* """ md_path = output_dir / "comparison_report.md" md_path.write_text(md) print(f" Markdown report: {md_path}") def main(): parser = argparse.ArgumentParser(description="Generate PyTorch vs RustyTorch++ comparison report") parser.add_argument("--output-dir", type=str, required=True, help="Output directory") parser.add_argument("--system-info", type=str, help="System info JSON file") parser.add_argument("--pytorch-cpu", type=str, help="PyTorch CPU results JSON") parser.add_argument("--pytorch-gpu", type=str, help="PyTorch GPU results JSON") parser.add_argument("--rust-cpu", type=str, help="Rust CPU Criterion output") parser.add_argument("--rust-gpu", type=str, help="Rust GPU Criterion output") args = parser.parse_args() output_dir = Path(args.output_dir) output_dir.mkdir(parents=True, exist_ok=True) # Load system info system_info = {} if args.system_info: system_info = load_system_info(Path(args.system_info)) # Parse benchmark results pytorch_cpu = [] pytorch_gpu = [] rust_cpu = [] rust_gpu = [] if args.pytorch_cpu: pytorch_cpu = parse_pytorch_json(Path(args.pytorch_cpu)) print(f" Parsed {len(pytorch_cpu)} PyTorch CPU results") if args.pytorch_gpu: pytorch_gpu = parse_pytorch_json(Path(args.pytorch_gpu)) print(f" Parsed {len(pytorch_gpu)} PyTorch GPU results") if args.rust_cpu: rust_cpu = parse_criterion_output(Path(args.rust_cpu)) print(f" Parsed {len(rust_cpu)} Rust CPU results") if args.rust_gpu: rust_gpu = parse_criterion_output(Path(args.rust_gpu)) print(f" Parsed {len(rust_gpu)} Rust GPU results") # Generate reports generate_html_report(output_dir, system_info, pytorch_cpu, pytorch_gpu, rust_cpu, rust_gpu) generate_markdown_report(output_dir, system_info, pytorch_cpu, pytorch_gpu, rust_cpu, rust_gpu) if __name__ == "__main__": main()