#!/usr/bin/env python3 """ Rove Typed HIR -> MIR bottleneck benchmark. Run from the Rove repository root: python rove_hir_mir_benchmark.py --repetitions 25 Optional GC comparison: python rove_hir_mir_benchmark.py --repetitions 25 --gc-mode both What it measures: - Current normal Python Stage-0 compile path (compile_source -> HIR -> current backend) - Isolated HIR -> MIR lowering - MIR verification only - Estimated current-pipeline + MIR overhead - Same measurements with Python cyclic GC disabled, if requested Important: - lower_hir_to_mir() ALREADY calls verify_mir() internally in the current Rove tree. - Therefore "HIR -> MIR" is the real current lowering cost including its built-in verification. - "MIR verify only" is measured separately only to show verifier cost if run again. """ from __future__ import annotations import argparse import gc import json import statistics import sys import time from pathlib import Path ROOT = Path(__file__).resolve().parent CORPUS = ROOT / "tests" / "fixtures" / "compiler_benchmark.json" def median_ms(values): return statistics.median(values) def p95_ms(values): if len(values) < 2: return values[0] xs = sorted(values) idx = min(len(xs) - 1, max(0, round((len(xs) - 1) * 0.95))) return xs[idx] def timed_call(fn): start = time.perf_counter_ns() value = fn() elapsed_ms = (time.perf_counter_ns() - start) / 1_000_000 return value, elapsed_ms def set_gc_mode(mode: str): if mode == "normal": gc.enable() elif mode == "disabled": gc.disable() else: raise ValueError(mode) def benchmark_source(source_path: Path, target: str, repetitions: int, gc_mode: str): from src import api from src.mir import lower_hir_to_mir, verify_mir set_gc_mode(gc_mode) source = source_path.read_text(encoding="utf-8") compiler = api.RoveCompiler(str(source_path.parent)) # Warm normal pipeline. warm = compiler.compile_source(source, filename=str(source_path), target=target) if not warm.success or warm.hir is None: raise RuntimeError(f"Warmup failed for {source_path}: {warm.diagnostics}") # Warm MIR path too. warm_mir = lower_hir_to_mir(warm.hir) verify_mir(warm_mir) normal_compile_ms = [] hir_to_mir_ms = [] verify_again_ms = [] for _ in range(repetitions): # 1. Current real/normal Stage-0 compile path. result, elapsed = timed_call( lambda: compiler.compile_source( source, filename=str(source_path), target=target, ) ) if not result.success or result.hir is None: raise RuntimeError(f"Compile failed for {source_path}: {result.diagnostics}") normal_compile_ms.append(elapsed) # 2. Isolated HIR -> MIR path from the HIR we just produced. # NOTE: current lower_hir_to_mir() includes verify_mir() internally. mir, elapsed = timed_call(lambda: lower_hir_to_mir(result.hir)) hir_to_mir_ms.append(elapsed) # 3. Measure an extra verifier pass separately. _, elapsed = timed_call(lambda: verify_mir(mir)) verify_again_ms.append(elapsed) normal_med = median_ms(normal_compile_ms) mir_med = median_ms(hir_to_mir_ms) verify_med = median_ms(verify_again_ms) combined = normal_med + mir_med overhead_pct = (mir_med / normal_med * 100.0) if normal_med else 0.0 combined_growth_pct = ((combined / normal_med) - 1.0) * 100.0 if normal_med else 0.0 return { "source": source_path.relative_to(ROOT).as_posix(), "gcMode": gc_mode, "repetitions": repetitions, "normalCompile": { "medianMs": normal_med, "p95Ms": p95_ms(normal_compile_ms), "runsMs": normal_compile_ms, }, "hirToMirIncludingInternalVerify": { "medianMs": mir_med, "p95Ms": p95_ms(hir_to_mir_ms), "runsMs": hir_to_mir_ms, }, "extraMirVerifyOnly": { "medianMs": verify_med, "p95Ms": p95_ms(verify_again_ms), "runsMs": verify_again_ms, }, "derived": { "mirCostAsPercentOfCurrentCompile": overhead_pct, "currentPlusMirMedianMs": combined, "currentPlusMirGrowthPercent": combined_growth_pct, }, } def aggregate(results): normal = [r["normalCompile"]["medianMs"] for r in results] mir = [r["hirToMirIncludingInternalVerify"]["medianMs"] for r in results] verify = [r["extraMirVerifyOnly"]["medianMs"] for r in results] normal_sum = sum(normal) mir_sum = sum(mir) verify_sum = sum(verify) return { "normalCompileCorpusMedianSumMs": normal_sum, "hirToMirCorpusMedianSumMs": mir_sum, "extraMirVerifyCorpusMedianSumMs": verify_sum, "mirCostAsPercentOfNormalCorpus": (mir_sum / normal_sum * 100.0) if normal_sum else 0.0, "normalPlusMirCorpusMs": normal_sum + mir_sum, "normalPlusMirGrowthPercent": (mir_sum / normal_sum * 100.0) if normal_sum else 0.0, } def print_table(mode_results): print() print("Rove HIR/MIR bottleneck benchmark") print("=" * 96) print( f"{'source':38} {'GC':9} {'normal ms':>11} {'HIR->MIR ms':>12} " f"{'verify ms':>10} {'MIR/current':>12}" ) print("-" * 96) for r in mode_results: print( f"{r['source'][-38:]:38} " f"{r['gcMode']:9} " f"{r['normalCompile']['medianMs']:11.3f} " f"{r['hirToMirIncludingInternalVerify']['medianMs']:12.3f} " f"{r['extraMirVerifyOnly']['medianMs']:10.3f} " f"{r['derived']['mirCostAsPercentOfCurrentCompile']:11.1f}%" ) print("-" * 96) def main(): parser = argparse.ArgumentParser() parser.add_argument("--repetitions", type=int, default=25) parser.add_argument( "--gc-mode", choices=("normal", "disabled", "both"), default="both", help="Compare normal CPython GC behavior with cyclic GC disabled.", ) parser.add_argument( "--output", type=Path, default=ROOT / "build" / "hir-mir-benchmark.json", ) args = parser.parse_args() if not 3 <= args.repetitions <= 500: parser.error("--repetitions must be between 3 and 500") if not CORPUS.exists(): raise SystemExit( f"Run this script from the Rove repository root. Missing: {CORPUS}" ) manifest = json.loads(CORPUS.read_text(encoding="utf-8")) target = manifest.get("target", "cpp") sources = [(ROOT / p).resolve() for p in manifest["sources"]] modes = ["normal", "disabled"] if args.gc_mode == "both" else [args.gc_mode] all_results = [] mode_summaries = {} original_gc = gc.isenabled() try: for mode in modes: results = [ benchmark_source(path, target, args.repetitions, mode) for path in sources ] all_results.extend(results) mode_summaries[mode] = aggregate(results) print_table(results) s = mode_summaries[mode] print( f"{mode}: corpus normal={s['normalCompileCorpusMedianSumMs']:.3f} ms, " f"HIR->MIR={s['hirToMirCorpusMedianSumMs']:.3f} ms, " f"MIR/current={s['mirCostAsPercentOfNormalCorpus']:.1f}%" ) finally: if original_gc: gc.enable() else: gc.disable() comparison = None if "normal" in mode_summaries and "disabled" in mode_summaries: n = mode_summaries["normal"]["normalCompileCorpusMedianSumMs"] d = mode_summaries["disabled"]["normalCompileCorpusMedianSumMs"] nm = mode_summaries["normal"]["hirToMirCorpusMedianSumMs"] dm = mode_summaries["disabled"]["hirToMirCorpusMedianSumMs"] comparison = { "normalCompileGcDisabledSpeedupPercent": ((n - d) / n * 100.0) if n else 0.0, "hirToMirGcDisabledSpeedupPercent": ((nm - dm) / nm * 100.0) if nm else 0.0, } report = { "schemaVersion": 1, "purpose": "Measure Python Stage-0 current compile path versus Typed HIR -> MIR lowering cost.", "python": sys.version, "target": target, "notes": [ "Uses the existing tests/fixtures/compiler_benchmark.json corpus.", "Normal compile uses RoveCompiler.compile_source().", "HIR->MIR uses src.mir.lower_hir_to_mir().", "Current lower_hir_to_mir() already invokes verify_mir() internally.", "The separate verify-only measurement is intentionally an extra verifier pass.", "GC-disabled mode disables CPython cyclic GC only; reference counting remains active.", "This is a Python Stage-0 benchmark, not native rovec performance.", ], "summaries": mode_summaries, "gcComparison": comparison, "results": all_results, } args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") print() print(f"Wrote: {args.output}") if comparison: print( f"GC disabled speedup: normal compile " f"{comparison['normalCompileGcDisabledSpeedupPercent']:.1f}% | " f"HIR->MIR {comparison['hirToMirGcDisabledSpeedupPercent']:.1f}%" ) if __name__ == "__main__": main()