Files
zed/script/gpui-bench-compare
Anthony Eid 4ce4b7b567 gpui: Move a Div's Interactivity out of line (#66719)
## Summary

Tessellating vector paths dominates the randomized element tree's
frames, hiding layout and the rest of the frame. This adds a
`with_paths(false)` option (the canvas stays, so layout is unchanged),
`PathlessTree/full refresh` and `PathlessTree/element style` benchmark
groups, and adds the path-free full refresh to
`script/gpui-bench-compare`'s core suite.

It also adds a random text mutation to the randomized element tree, so
its tests cover text whose measured size changes (including text that
wraps and text that's removed). Generated trees are unchanged, but the
extra mutation kind changes which mutations each seed applies.

These PRs are stacked on this one and use its path-free groups for their
benchmark numbers:

- #66723 gpui: Move a Div's Interactivity out of line (merged into this
branch)
- #66720 gpui: Only take a line wrapper when truncating text
- #66721 gpui: Compute tab stop order only when moving focus
- #66722 gpui: Trim per-line and per-glyph work in the text system

Release Notes:

- N/A
- [GPUI] Improve randomize element tree test
2026-11-09 04:15:58 +00:00

370 lines
18 KiB
Python
Executable File

#!/usr/bin/env python3
"""Build two revisions once, then alternate bounded GPUI benchmark runs.
Usage: script/gpui-bench-compare BASELINE CANDIDATE
script/gpui-bench-compare BASELINE CANDIDATE --filter 'TextPanels/'
The invoking checkout's three benchmark files, randomized tree fixture and bench
macro are used for BOTH revisions, so both revisions must provide the GPUI
benchmark APIs they call. Sources, build artifacts, binaries, raw logs, Criterion
samples and summary.json persist under target/gpui-bench-compare. Nothing
switches the invoking checkout or publishes a branch.
The default core suite keeps the mixed workload (plain and with cached views),
five full-refresh tree shapes with and without paths, element style, cached
element style, pointer hover, warm installed-font text, text first frame, three
scrolling workloads, random and patterned primitive painting, and two inspector
workloads. --full selects every variant; --slow restores the benchmarks' own
Criterion settings (including inspector's five-second measurement).
--repetitions controls independent runs, not random seeds. Use --seed to
compare another fixed tree.
Foreground retired instructions are primary. Linux needs perf access
(CAP_PERFMON or perf_event_paranoid <= 2); macOS needs Apple Silicon. Frames run
the production pipeline through the platform's headless GPU renderer, so they
include scene encoding and GPU submission; GPU execution itself runs on other
threads and isn't counted. Wall-time medians include calibration samples; raw
logs retain frame metrics. Spread is the full range of independent run medians
divided by their median, not a confidence interval. Warnings also flag wide
within-run sample ranges; even slow settings can drift, so inspect raw samples
before judging small changes. Do not run competing benchmarks on the same machine.
Requires Python 3.12+ on Linux or macOS.
"""
import argparse
import fcntl
import hashlib
import json
import os
from pathlib import Path
import re
import shutil
import signal
import statistics
import subprocess
import sys
import tarfile
import time
TARGETS = ("randomized_element_tree", "primitives", "inspector_render")
FIXTURE_FILES = tuple(
Path("crates/gpui_platform/benches") / f"{target}.rs" for target in TARGETS
) + (
Path("crates/gpui_macros/src/bench.rs"),
Path("crates/gpui/src/randomized_element_tree.rs"),
)
CORE = {
"randomized_element_tree": (
r"^((Randomized|Cached)Tree/mixed/tree/mixed/|"
r"RandomizedTree/full refresh/tree/|"
r"PathlessTree/full refresh/tree/|"
r"RandomizedTree/element style/tree/mixed/|"
r"CachedTree/element style/tree/mixed/|"
r"RandomizedTree/hover/tree/mixed/|"
r"Scroll/workload/|"
r"TextPanels/installed font two updates/panels/32$|"
r"TextPanels/first populated installed font/panels/32$)"
),
# Overlapping paint cases, including the random ones with large paths and text,
# primarily stress GPU overdraw, not draw ordering; GPU queue backpressure makes
# their instruction counts vary by up to ~16% within a run.
"primitives": (
r"^Primitives/(scene/[a-z]+-1024(-seed-\d+)?|paint/case/grid-1024)$"
),
"inspector_render": (
r"^closed_inspector_(grid/case/1000-nodes|panel/case/500-rows)-0-cycles$"
),
}
FAST = [
"--warm-up-time", "0.05",
"--measurement-time", "0.3",
"--sample-size", "10",
"--nresamples", "1000",
]
def run(command, *, cwd, environment, log, timeout):
started = time.monotonic()
with log.open("w") as output:
# A session of its own lets a timeout or interrupt stop the whole process tree:
# killing only Cargo would leave its compiler and build-script children running
# after this script exits and releases the comparison lock.
process = subprocess.Popen(
command, cwd=cwd, env=environment, stdout=output,
stderr=subprocess.STDOUT, start_new_session=True,
)
try:
returncode = process.wait(timeout=timeout)
except BaseException as error:
os.killpg(process.pid, signal.SIGKILL)
process.wait()
if isinstance(error, subprocess.TimeoutExpired):
raise RuntimeError(f"{error}; see {log}") from error
raise
if returncode != 0:
raise RuntimeError(f"{command[0]} exited with status {returncode}; see {log}")
return time.monotonic() - started
def git(root, *arguments):
return subprocess.check_output(["git", "-C", str(root), *arguments], text=True).strip()
def prepare(root, cache, revision, features, timeout, environment):
commit = git(root, "rev-parse", "--verify", f"{revision}^{{commit}}")
source = cache / "sources" / commit
source.mkdir(parents=True, exist_ok=True)
if not (source / ".archive-complete").exists():
archive = cache / f"{commit}.tar"
with archive.open("wb") as output:
subprocess.run(["git", "-C", str(root), "archive", commit],
stdout=output, check=True, timeout=timeout)
with tarfile.open(archive) as contents:
contents.extractall(source, filter="data")
(source / ".archive-complete").touch()
digest = hashlib.sha256()
for relative in FIXTURE_FILES:
content = (root / relative).read_bytes()
digest.update(str(relative).encode())
digest.update(content)
destination = source / relative
if not destination.exists() or destination.read_bytes() != content:
destination.write_bytes(content)
build_key = hashlib.sha256(
(commit + digest.hexdigest() + features).encode()
).hexdigest()[:20]
artifacts = cache / "binaries" / build_key
artifacts.mkdir(parents=True, exist_ok=True)
# Cargo's workspace fingerprints use mtimes, not source contents. Archives
# from different commits can otherwise reuse the preceding revision's binary.
environment = dict(environment, CARGO_TARGET_DIR=str(cache / "build" / commit))
feature_log = artifacts / "features.log"
run(
["cargo", "tree", "--locked", "-p", "gpui_platform", "--features", features,
"-e", "normal,build,dev,features"],
cwd=source, environment=environment, log=feature_log, timeout=timeout,
)
if 'feature "test-support"' in feature_log.read_text():
raise RuntimeError(f"benchmark graph enables test-support: {feature_log}")
command = [
"cargo", "bench", "--locked", "-p", "gpui_platform", "--features", features,
"--profile", "release-fast", "--no-run", "--message-format=json",
]
for target in TARGETS:
command += ["--bench", target]
elapsed = run(command, cwd=source, environment=environment,
log=artifacts / "build.log", timeout=timeout)
binaries = {}
for line in (artifacts / "build.log").read_text().splitlines():
try:
message = json.loads(line)
except json.JSONDecodeError:
continue
if message.get("reason") != "compiler-artifact" or not message.get("executable"):
continue
name = message["target"]["name"]
if name in TARGETS:
# Cargo may overwrite these paths on the next revision's build.
destination = artifacts / name
shutil.copy2(message["executable"], destination)
binaries[name] = destination
if set(binaries) != set(TARGETS):
raise RuntimeError(f"missing benchmark executable; see {artifacts / 'build.log'}")
binary_hashes = {}
for target, binary in binaries.items():
with binary.open("rb") as executable:
binary_hashes[target] = hashlib.file_digest(executable, "sha256").hexdigest()
return {
"revision": revision, "commit": commit, "build_seconds": elapsed,
"gpui_tree": git(root, "rev-parse", f"{commit}:crates/gpui"),
"binary_sha256": binary_hashes,
"benchmark_sha256": digest.hexdigest(),
"lockfile_sha256": hashlib.sha256((source / "Cargo.lock").read_bytes()).hexdigest(),
"source": str(source), "binaries": binaries,
}
def verify_distinct_artifacts(revisions):
baseline, candidate = revisions
if baseline["gpui_tree"] != candidate["gpui_tree"]:
for target in TARGETS:
if baseline["binary_sha256"][target] == candidate["binary_sha256"][target]:
raise RuntimeError(
f"{target} binaries are byte-identical despite different crates/gpui trees; "
"refusing a potentially contaminated benchmark comparison"
)
def measure(binary, identifier, directory, environment, timeout, slow):
directory.mkdir(parents=True, exist_ok=True)
environment = dict(environment, CRITERION_HOME=str(directory / "criterion"))
command = [str(binary), "--bench", "^" + re.escape(identifier) + "$", "--noplot"]
if not slow:
command += FAST
elapsed = run(command, cwd=directory, environment=environment,
log=directory / "run.log", timeout=timeout)
samples = list((directory / "criterion").glob("**/new/sample.json"))
if len(samples) != 1:
raise RuntimeError(f"expected one measured case, got {len(samples)}: {directory}")
sample = json.loads(samples[0].read_text())
counts = [total / count for total, count in zip(sample["times"], sample["iters"])]
wall = re.search(
r"wall time per iteration: median ([\d.]+) (\S+) \(min ([\d.]+), max ([\d.]+)",
(directory / "run.log").read_text(),
)
if wall is None:
raise RuntimeError(f"missing wall-time secondary: {directory / 'run.log'}")
units = {"ns": 1e-9, "µs": 1e-6, "us": 1e-6, "ms": 1e-3, "s": 1}
return {
"instructions": statistics.median(counts),
"sample_min": min(counts), "sample_max": max(counts),
"wall_seconds": float(wall[1]) * units[wall[2]],
"elapsed_seconds": elapsed, "iterations": sum(sample["iters"]),
}
def spread(values):
return (max(values) - min(values)) / statistics.median(values) * 100
def main():
started = time.monotonic()
parser = argparse.ArgumentParser(description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("baseline")
parser.add_argument("candidate")
parser.add_argument("--filter", help="Criterion regex, applied to all three binaries")
parser.add_argument("--full", action="store_true")
parser.add_argument("--slow", action="store_true")
parser.add_argument("--repetitions", type=int, default=3)
parser.add_argument("--seed", type=int, default=0)
# Text benchmarks need the platform's real text system: x11 on Linux, font-kit on macOS.
parser.add_argument("--features", default={"linux": "bench-support,x11",
"darwin": "bench-support,font-kit"}.get(
sys.platform, "bench-support"))
parser.add_argument("--timeout", type=int, default=300,
help="hard timeout for EACH build/run, at most 300 seconds")
parser.add_argument("--cache", type=Path)
args = parser.parse_args()
if not 1 <= args.timeout <= 300 or args.repetitions < 1 or args.seed < 0:
parser.error("timeout must be 1..300, repetitions positive and seed nonnegative")
if args.filter and args.full:
parser.error("--filter and --full are mutually exclusive")
if "bench-support" not in args.features.split(",") or "test-support" in args.features:
parser.error("features must include bench-support and must not include test-support")
root = Path(git(Path(__file__).resolve().parent, "rev-parse", "--show-toplevel"))
cache = (args.cache or root / "target/gpui-bench-compare").resolve()
cache.mkdir(parents=True, exist_ok=True)
lock = (cache / "compare.lock").open("w")
try:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
except BlockingIOError as error:
raise RuntimeError(f"another comparison is using {cache}") from error
output = cache / "runs" / time.strftime("%Y%m%d-%H%M%S")
output.mkdir(parents=True, exist_ok=False)
environment = dict(os.environ, BENCH_MEASUREMENT="foreground-instructions",
SEED=str(args.seed), ITERATIONS="1")
environment["PATH"] = str(Path.home() / ".cargo/bin") + os.pathsep + environment["PATH"]
revisions = []
for revision in [args.baseline, args.candidate]:
print(f"Building {revision} (once)...", flush=True)
revisions.append(prepare(root, cache, revision, args.features, args.timeout, environment))
verify_distinct_artifacts(revisions)
if revisions[0]["lockfile_sha256"] != revisions[1]["lockfile_sha256"]:
print("NOTE: revisions have different Cargo.lock files; dependency changes may affect results.")
cases = {}
for target in TARGETS:
listings = []
pattern = args.filter if args.filter else "" if args.full else CORE[target]
for index, revision in enumerate(revisions):
log = output / f"list-{index}-{target}.log"
run([str(revision["binaries"][target]), "--list", pattern],
cwd=output, environment=environment, log=log, timeout=args.timeout)
listings.append([line.removesuffix(": benchmark") for line in log.read_text().splitlines()
if line.endswith(": benchmark")])
if listings[0] != listings[1]:
raise RuntimeError(f"benchmark IDs differ for {target}")
cases[target] = listings[0]
if not any(cases.values()):
raise RuntimeError("filter selected no benchmarks")
# Verify all selected paths once; exercise Criterion's quick path before sampling.
for index, revision in enumerate(revisions):
for target, identifiers in cases.items():
if not identifiers:
continue
binary = revision["binaries"][target]
pattern = "^(" + "|".join(re.escape(value) for value in identifiers) + ")$"
for mode, selection in [("--test", pattern),
("--quick", "^" + re.escape(identifiers[0]) + "$")]:
run([str(binary), "--bench", selection, mode, "--noplot",
"--measurement-time", "0.3"], cwd=output, environment=environment,
log=output / f"smoke-{index}-{target}{mode}.log", timeout=args.timeout)
results = {}
print("Measuring sequentially; instructions are foreground retired instructions per frame/operation.",
flush=True)
print(f"{'benchmark':76} {'base M':>9} {'new M':>9} {'change':>9} "
f"{'spread B/N':>14} {'wall ms B/N':>17} {'run s B/N':>13}", flush=True)
for target, identifiers in cases.items():
for identifier in identifiers:
pair = [[], []]
case_number = len(results)
for repetition in range(args.repetitions):
for index in ([0, 1] if repetition % 2 == 0 else [1, 0]):
revision = revisions[index]
directory = output / f"{case_number:03}-{repetition}-{index}"
pair[index].append(measure(revision["binaries"][target], identifier, directory,
environment, args.timeout, args.slow))
results[identifier] = pair
instruction_runs = [[value["instructions"] for value in side] for side in pair]
baseline, candidate = map(statistics.median, instruction_runs)
walls = [statistics.median(value["wall_seconds"] for value in side) * 1000 for side in pair]
durations = [statistics.median(value["elapsed_seconds"] for value in side) for side in pair]
noise = (f"{spread(instruction_runs[0]):5.2f}/{spread(instruction_runs[1]):5.2f}%"
if args.repetitions > 1 else " n/a")
print(f"{identifier:76} {baseline / 1e6:9.3f} {candidate / 1e6:9.3f} "
f"{(candidate / baseline - 1) * 100:+8.2f}% "
f"{noise} "
f"{walls[0]:7.3f}/{walls[1]:7.3f} "
f"{durations[0]:5.2f}/{durations[1]:5.2f}", flush=True)
if args.repetitions > 1 and any(spread(values) > 1 for values in instruction_runs):
print(" NOTE: >1% instruction spread; confirm this case with --slow before judging a change.",
flush=True)
sample_ranges = [
max((value["sample_max"] - value["sample_min"]) / value["instructions"] * 100
for value in side)
for side in pair
]
if any(value > 1 for value in sample_ranges):
print(f" NOTE: within-run instruction sample range B/N "
f"{sample_ranges[0]:.2f}/{sample_ranges[1]:.2f}%; "
"inspect raw samples for drift/outliers before judging a change.", flush=True)
summary = {
"configuration": vars(args) | {"cache": str(cache)},
"revisions": [{key: value for key, value in revision.items() if key != "binaries"}
for revision in revisions],
"results": results,
"cases": cases,
"environment": {name: value for name, value in environment.items()
if name in ("BENCH_MEASUREMENT", "SEED", "ITERATIONS",
"RUSTFLAGS")},
}
(output / "summary.json").write_text(json.dumps(summary, indent=2, default=str) + "\n")
for index, revision in enumerate(revisions):
seconds = sum(value["elapsed_seconds"] for pair in results.values() for value in pair[index])
print(f"{revision['commit'][:12]}: build {revision['build_seconds']:.2f}s, "
f"measured processes {seconds:.2f}s ({args.repetitions} runs/case)")
for target, identifiers in cases.items():
seconds = sum(value["elapsed_seconds"] for identifier in identifiers
for value in results[identifier][index])
print(f" {target}: {seconds:.2f}s")
print(f"Total including builds, discovery, smoke and quick checks: {time.monotonic() - started:.2f}s")
print(f"Raw samples, frame reports and summary: {output}")
if __name__ == "__main__":
try:
main()
except (RuntimeError, subprocess.CalledProcessError) as error:
sys.exit(str(error))