#!/usr/bin/env python3 """Build two revisions once, then alternate bounded GPUI benchmark runs. Usage: script/gpui-bench-compare BASELINE CANDIDATE script/gpui-bench-compare BASELINE CANDIDATE --filter 'TextPanels/' The invoking checkout's three benchmark files, randomized tree fixture and bench macro are used for BOTH revisions, so both revisions must provide the GPUI benchmark APIs they call. Sources, build artifacts, binaries, raw logs, Criterion samples and summary.json persist under target/gpui-bench-compare. Nothing switches the invoking checkout or publishes a branch. The default core suite keeps the mixed workload (plain and with cached views), five full-refresh tree shapes with and without paths, element style, cached element style, pointer hover, warm installed-font text, text first frame, three scrolling workloads, random and patterned primitive painting, and two inspector workloads. --full selects every variant; --slow restores the benchmarks' own Criterion settings (including inspector's five-second measurement). --repetitions controls independent runs, not random seeds. Use --seed to compare another fixed tree. Foreground retired instructions are primary. Linux needs perf access (CAP_PERFMON or perf_event_paranoid <= 2); macOS needs Apple Silicon. Frames run the production pipeline through the platform's headless GPU renderer, so they include scene encoding and GPU submission; GPU execution itself runs on other threads and isn't counted. Wall-time medians include calibration samples; raw logs retain frame metrics. Spread is the full range of independent run medians divided by their median, not a confidence interval. Warnings also flag wide within-run sample ranges; even slow settings can drift, so inspect raw samples before judging small changes. Do not run competing benchmarks on the same machine. Requires Python 3.12+ on Linux or macOS. """ import argparse import fcntl import hashlib import json import os from pathlib import Path import re import shutil import signal import statistics import subprocess import sys import tarfile import time TARGETS = ("randomized_element_tree", "primitives", "inspector_render") FIXTURE_FILES = tuple( Path("crates/gpui_platform/benches") / f"{target}.rs" for target in TARGETS ) + ( Path("crates/gpui_macros/src/bench.rs"), Path("crates/gpui/src/randomized_element_tree.rs"), ) CORE = { "randomized_element_tree": ( r"^((Randomized|Cached)Tree/mixed/tree/mixed/|" r"RandomizedTree/full refresh/tree/|" r"PathlessTree/full refresh/tree/|" r"RandomizedTree/element style/tree/mixed/|" r"CachedTree/element style/tree/mixed/|" r"RandomizedTree/hover/tree/mixed/|" r"Scroll/workload/|" r"TextPanels/installed font two updates/panels/32$|" r"TextPanels/first populated installed font/panels/32$)" ), # Overlapping paint cases, including the random ones with large paths and text, # primarily stress GPU overdraw, not draw ordering; GPU queue backpressure makes # their instruction counts vary by up to ~16% within a run. "primitives": ( r"^Primitives/(scene/[a-z]+-1024(-seed-\d+)?|paint/case/grid-1024)$" ), "inspector_render": ( r"^closed_inspector_(grid/case/1000-nodes|panel/case/500-rows)-0-cycles$" ), } FAST = [ "--warm-up-time", "0.05", "--measurement-time", "0.3", "--sample-size", "10", "--nresamples", "1000", ] def run(command, *, cwd, environment, log, timeout): started = time.monotonic() with log.open("w") as output: # A session of its own lets a timeout or interrupt stop the whole process tree: # killing only Cargo would leave its compiler and build-script children running # after this script exits and releases the comparison lock. process = subprocess.Popen( command, cwd=cwd, env=environment, stdout=output, stderr=subprocess.STDOUT, start_new_session=True, ) try: returncode = process.wait(timeout=timeout) except BaseException as error: os.killpg(process.pid, signal.SIGKILL) process.wait() if isinstance(error, subprocess.TimeoutExpired): raise RuntimeError(f"{error}; see {log}") from error raise if returncode != 0: raise RuntimeError(f"{command[0]} exited with status {returncode}; see {log}") return time.monotonic() - started def git(root, *arguments): return subprocess.check_output(["git", "-C", str(root), *arguments], text=True).strip() def prepare(root, cache, revision, features, timeout, environment): commit = git(root, "rev-parse", "--verify", f"{revision}^{{commit}}") source = cache / "sources" / commit source.mkdir(parents=True, exist_ok=True) if not (source / ".archive-complete").exists(): archive = cache / f"{commit}.tar" with archive.open("wb") as output: subprocess.run(["git", "-C", str(root), "archive", commit], stdout=output, check=True, timeout=timeout) with tarfile.open(archive) as contents: contents.extractall(source, filter="data") (source / ".archive-complete").touch() digest = hashlib.sha256() for relative in FIXTURE_FILES: content = (root / relative).read_bytes() digest.update(str(relative).encode()) digest.update(content) destination = source / relative if not destination.exists() or destination.read_bytes() != content: destination.write_bytes(content) build_key = hashlib.sha256( (commit + digest.hexdigest() + features).encode() ).hexdigest()[:20] artifacts = cache / "binaries" / build_key artifacts.mkdir(parents=True, exist_ok=True) # Cargo's workspace fingerprints use mtimes, not source contents. Archives # from different commits can otherwise reuse the preceding revision's binary. environment = dict(environment, CARGO_TARGET_DIR=str(cache / "build" / commit)) feature_log = artifacts / "features.log" run( ["cargo", "tree", "--locked", "-p", "gpui_platform", "--features", features, "-e", "normal,build,dev,features"], cwd=source, environment=environment, log=feature_log, timeout=timeout, ) if 'feature "test-support"' in feature_log.read_text(): raise RuntimeError(f"benchmark graph enables test-support: {feature_log}") command = [ "cargo", "bench", "--locked", "-p", "gpui_platform", "--features", features, "--profile", "release-fast", "--no-run", "--message-format=json", ] for target in TARGETS: command += ["--bench", target] elapsed = run(command, cwd=source, environment=environment, log=artifacts / "build.log", timeout=timeout) binaries = {} for line in (artifacts / "build.log").read_text().splitlines(): try: message = json.loads(line) except json.JSONDecodeError: continue if message.get("reason") != "compiler-artifact" or not message.get("executable"): continue name = message["target"]["name"] if name in TARGETS: # Cargo may overwrite these paths on the next revision's build. destination = artifacts / name shutil.copy2(message["executable"], destination) binaries[name] = destination if set(binaries) != set(TARGETS): raise RuntimeError(f"missing benchmark executable; see {artifacts / 'build.log'}") binary_hashes = {} for target, binary in binaries.items(): with binary.open("rb") as executable: binary_hashes[target] = hashlib.file_digest(executable, "sha256").hexdigest() return { "revision": revision, "commit": commit, "build_seconds": elapsed, "gpui_tree": git(root, "rev-parse", f"{commit}:crates/gpui"), "binary_sha256": binary_hashes, "benchmark_sha256": digest.hexdigest(), "lockfile_sha256": hashlib.sha256((source / "Cargo.lock").read_bytes()).hexdigest(), "source": str(source), "binaries": binaries, } def verify_distinct_artifacts(revisions): baseline, candidate = revisions if baseline["gpui_tree"] != candidate["gpui_tree"]: for target in TARGETS: if baseline["binary_sha256"][target] == candidate["binary_sha256"][target]: raise RuntimeError( f"{target} binaries are byte-identical despite different crates/gpui trees; " "refusing a potentially contaminated benchmark comparison" ) def measure(binary, identifier, directory, environment, timeout, slow): directory.mkdir(parents=True, exist_ok=True) environment = dict(environment, CRITERION_HOME=str(directory / "criterion")) command = [str(binary), "--bench", "^" + re.escape(identifier) + "$", "--noplot"] if not slow: command += FAST elapsed = run(command, cwd=directory, environment=environment, log=directory / "run.log", timeout=timeout) samples = list((directory / "criterion").glob("**/new/sample.json")) if len(samples) != 1: raise RuntimeError(f"expected one measured case, got {len(samples)}: {directory}") sample = json.loads(samples[0].read_text()) counts = [total / count for total, count in zip(sample["times"], sample["iters"])] wall = re.search( r"wall time per iteration: median ([\d.]+) (\S+) \(min ([\d.]+), max ([\d.]+)", (directory / "run.log").read_text(), ) if wall is None: raise RuntimeError(f"missing wall-time secondary: {directory / 'run.log'}") units = {"ns": 1e-9, "µs": 1e-6, "us": 1e-6, "ms": 1e-3, "s": 1} return { "instructions": statistics.median(counts), "sample_min": min(counts), "sample_max": max(counts), "wall_seconds": float(wall[1]) * units[wall[2]], "elapsed_seconds": elapsed, "iterations": sum(sample["iters"]), } def spread(values): return (max(values) - min(values)) / statistics.median(values) * 100 def main(): started = time.monotonic() parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument("baseline") parser.add_argument("candidate") parser.add_argument("--filter", help="Criterion regex, applied to all three binaries") parser.add_argument("--full", action="store_true") parser.add_argument("--slow", action="store_true") parser.add_argument("--repetitions", type=int, default=3) parser.add_argument("--seed", type=int, default=0) # Text benchmarks need the platform's real text system: x11 on Linux, font-kit on macOS. parser.add_argument("--features", default={"linux": "bench-support,x11", "darwin": "bench-support,font-kit"}.get( sys.platform, "bench-support")) parser.add_argument("--timeout", type=int, default=300, help="hard timeout for EACH build/run, at most 300 seconds") parser.add_argument("--cache", type=Path) args = parser.parse_args() if not 1 <= args.timeout <= 300 or args.repetitions < 1 or args.seed < 0: parser.error("timeout must be 1..300, repetitions positive and seed nonnegative") if args.filter and args.full: parser.error("--filter and --full are mutually exclusive") if "bench-support" not in args.features.split(",") or "test-support" in args.features: parser.error("features must include bench-support and must not include test-support") root = Path(git(Path(__file__).resolve().parent, "rev-parse", "--show-toplevel")) cache = (args.cache or root / "target/gpui-bench-compare").resolve() cache.mkdir(parents=True, exist_ok=True) lock = (cache / "compare.lock").open("w") try: fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) except BlockingIOError as error: raise RuntimeError(f"another comparison is using {cache}") from error output = cache / "runs" / time.strftime("%Y%m%d-%H%M%S") output.mkdir(parents=True, exist_ok=False) environment = dict(os.environ, BENCH_MEASUREMENT="foreground-instructions", SEED=str(args.seed), ITERATIONS="1") environment["PATH"] = str(Path.home() / ".cargo/bin") + os.pathsep + environment["PATH"] revisions = [] for revision in [args.baseline, args.candidate]: print(f"Building {revision} (once)...", flush=True) revisions.append(prepare(root, cache, revision, args.features, args.timeout, environment)) verify_distinct_artifacts(revisions) if revisions[0]["lockfile_sha256"] != revisions[1]["lockfile_sha256"]: print("NOTE: revisions have different Cargo.lock files; dependency changes may affect results.") cases = {} for target in TARGETS: listings = [] pattern = args.filter if args.filter else "" if args.full else CORE[target] for index, revision in enumerate(revisions): log = output / f"list-{index}-{target}.log" run([str(revision["binaries"][target]), "--list", pattern], cwd=output, environment=environment, log=log, timeout=args.timeout) listings.append([line.removesuffix(": benchmark") for line in log.read_text().splitlines() if line.endswith(": benchmark")]) if listings[0] != listings[1]: raise RuntimeError(f"benchmark IDs differ for {target}") cases[target] = listings[0] if not any(cases.values()): raise RuntimeError("filter selected no benchmarks") # Verify all selected paths once; exercise Criterion's quick path before sampling. for index, revision in enumerate(revisions): for target, identifiers in cases.items(): if not identifiers: continue binary = revision["binaries"][target] pattern = "^(" + "|".join(re.escape(value) for value in identifiers) + ")$" for mode, selection in [("--test", pattern), ("--quick", "^" + re.escape(identifiers[0]) + "$")]: run([str(binary), "--bench", selection, mode, "--noplot", "--measurement-time", "0.3"], cwd=output, environment=environment, log=output / f"smoke-{index}-{target}{mode}.log", timeout=args.timeout) results = {} print("Measuring sequentially; instructions are foreground retired instructions per frame/operation.", flush=True) print(f"{'benchmark':76} {'base M':>9} {'new M':>9} {'change':>9} " f"{'spread B/N':>14} {'wall ms B/N':>17} {'run s B/N':>13}", flush=True) for target, identifiers in cases.items(): for identifier in identifiers: pair = [[], []] case_number = len(results) for repetition in range(args.repetitions): for index in ([0, 1] if repetition % 2 == 0 else [1, 0]): revision = revisions[index] directory = output / f"{case_number:03}-{repetition}-{index}" pair[index].append(measure(revision["binaries"][target], identifier, directory, environment, args.timeout, args.slow)) results[identifier] = pair instruction_runs = [[value["instructions"] for value in side] for side in pair] baseline, candidate = map(statistics.median, instruction_runs) walls = [statistics.median(value["wall_seconds"] for value in side) * 1000 for side in pair] durations = [statistics.median(value["elapsed_seconds"] for value in side) for side in pair] noise = (f"{spread(instruction_runs[0]):5.2f}/{spread(instruction_runs[1]):5.2f}%" if args.repetitions > 1 else " n/a") print(f"{identifier:76} {baseline / 1e6:9.3f} {candidate / 1e6:9.3f} " f"{(candidate / baseline - 1) * 100:+8.2f}% " f"{noise} " f"{walls[0]:7.3f}/{walls[1]:7.3f} " f"{durations[0]:5.2f}/{durations[1]:5.2f}", flush=True) if args.repetitions > 1 and any(spread(values) > 1 for values in instruction_runs): print(" NOTE: >1% instruction spread; confirm this case with --slow before judging a change.", flush=True) sample_ranges = [ max((value["sample_max"] - value["sample_min"]) / value["instructions"] * 100 for value in side) for side in pair ] if any(value > 1 for value in sample_ranges): print(f" NOTE: within-run instruction sample range B/N " f"{sample_ranges[0]:.2f}/{sample_ranges[1]:.2f}%; " "inspect raw samples for drift/outliers before judging a change.", flush=True) summary = { "configuration": vars(args) | {"cache": str(cache)}, "revisions": [{key: value for key, value in revision.items() if key != "binaries"} for revision in revisions], "results": results, "cases": cases, "environment": {name: value for name, value in environment.items() if name in ("BENCH_MEASUREMENT", "SEED", "ITERATIONS", "RUSTFLAGS")}, } (output / "summary.json").write_text(json.dumps(summary, indent=2, default=str) + "\n") for index, revision in enumerate(revisions): seconds = sum(value["elapsed_seconds"] for pair in results.values() for value in pair[index]) print(f"{revision['commit'][:12]}: build {revision['build_seconds']:.2f}s, " f"measured processes {seconds:.2f}s ({args.repetitions} runs/case)") for target, identifiers in cases.items(): seconds = sum(value["elapsed_seconds"] for identifier in identifiers for value in results[identifier][index]) print(f" {target}: {seconds:.2f}s") print(f"Total including builds, discovery, smoke and quick checks: {time.monotonic() - started:.2f}s") print(f"Raw samples, frame reports and summary: {output}") if __name__ == "__main__": try: main() except (RuntimeError, subprocess.CalledProcessError) as error: sys.exit(str(error))