mirror of
https://github.com/zed-industries/zed.git
synced 2026-11-19 15:10:34 +01:00
4ce4b7b567
## Summary Tessellating vector paths dominates the randomized element tree's frames, hiding layout and the rest of the frame. This adds a `with_paths(false)` option (the canvas stays, so layout is unchanged), `PathlessTree/full refresh` and `PathlessTree/element style` benchmark groups, and adds the path-free full refresh to `script/gpui-bench-compare`'s core suite. It also adds a random text mutation to the randomized element tree, so its tests cover text whose measured size changes (including text that wraps and text that's removed). Generated trees are unchanged, but the extra mutation kind changes which mutations each seed applies. These PRs are stacked on this one and use its path-free groups for their benchmark numbers: - #66723 gpui: Move a Div's Interactivity out of line (merged into this branch) - #66720 gpui: Only take a line wrapper when truncating text - #66721 gpui: Compute tab stop order only when moving focus - #66722 gpui: Trim per-line and per-glyph work in the text system Release Notes: - N/A - [GPUI] Improve randomize element tree test
370 lines
18 KiB
Python
Executable File
370 lines
18 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Build two revisions once, then alternate bounded GPUI benchmark runs.
|
|
|
|
Usage: script/gpui-bench-compare BASELINE CANDIDATE
|
|
script/gpui-bench-compare BASELINE CANDIDATE --filter 'TextPanels/'
|
|
|
|
The invoking checkout's three benchmark files, randomized tree fixture and bench
|
|
macro are used for BOTH revisions, so both revisions must provide the GPUI
|
|
benchmark APIs they call. Sources, build artifacts, binaries, raw logs, Criterion
|
|
samples and summary.json persist under target/gpui-bench-compare. Nothing
|
|
switches the invoking checkout or publishes a branch.
|
|
|
|
The default core suite keeps the mixed workload (plain and with cached views),
|
|
five full-refresh tree shapes with and without paths, element style, cached
|
|
element style, pointer hover, warm installed-font text, text first frame, three
|
|
scrolling workloads, random and patterned primitive painting, and two inspector
|
|
workloads. --full selects every variant; --slow restores the benchmarks' own
|
|
Criterion settings (including inspector's five-second measurement).
|
|
--repetitions controls independent runs, not random seeds. Use --seed to
|
|
compare another fixed tree.
|
|
|
|
Foreground retired instructions are primary. Linux needs perf access
|
|
(CAP_PERFMON or perf_event_paranoid <= 2); macOS needs Apple Silicon. Frames run
|
|
the production pipeline through the platform's headless GPU renderer, so they
|
|
include scene encoding and GPU submission; GPU execution itself runs on other
|
|
threads and isn't counted. Wall-time medians include calibration samples; raw
|
|
logs retain frame metrics. Spread is the full range of independent run medians
|
|
divided by their median, not a confidence interval. Warnings also flag wide
|
|
within-run sample ranges; even slow settings can drift, so inspect raw samples
|
|
before judging small changes. Do not run competing benchmarks on the same machine.
|
|
Requires Python 3.12+ on Linux or macOS.
|
|
"""
|
|
|
|
import argparse
|
|
import fcntl
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import shutil
|
|
import signal
|
|
import statistics
|
|
import subprocess
|
|
import sys
|
|
import tarfile
|
|
import time
|
|
|
|
|
|
TARGETS = ("randomized_element_tree", "primitives", "inspector_render")
|
|
FIXTURE_FILES = tuple(
|
|
Path("crates/gpui_platform/benches") / f"{target}.rs" for target in TARGETS
|
|
) + (
|
|
Path("crates/gpui_macros/src/bench.rs"),
|
|
Path("crates/gpui/src/randomized_element_tree.rs"),
|
|
)
|
|
CORE = {
|
|
"randomized_element_tree": (
|
|
r"^((Randomized|Cached)Tree/mixed/tree/mixed/|"
|
|
r"RandomizedTree/full refresh/tree/|"
|
|
r"PathlessTree/full refresh/tree/|"
|
|
r"RandomizedTree/element style/tree/mixed/|"
|
|
r"CachedTree/element style/tree/mixed/|"
|
|
r"RandomizedTree/hover/tree/mixed/|"
|
|
r"Scroll/workload/|"
|
|
r"TextPanels/installed font two updates/panels/32$|"
|
|
r"TextPanels/first populated installed font/panels/32$)"
|
|
),
|
|
# Overlapping paint cases, including the random ones with large paths and text,
|
|
# primarily stress GPU overdraw, not draw ordering; GPU queue backpressure makes
|
|
# their instruction counts vary by up to ~16% within a run.
|
|
"primitives": (
|
|
r"^Primitives/(scene/[a-z]+-1024(-seed-\d+)?|paint/case/grid-1024)$"
|
|
),
|
|
"inspector_render": (
|
|
r"^closed_inspector_(grid/case/1000-nodes|panel/case/500-rows)-0-cycles$"
|
|
),
|
|
}
|
|
FAST = [
|
|
"--warm-up-time", "0.05",
|
|
"--measurement-time", "0.3",
|
|
"--sample-size", "10",
|
|
"--nresamples", "1000",
|
|
]
|
|
|
|
|
|
def run(command, *, cwd, environment, log, timeout):
|
|
started = time.monotonic()
|
|
with log.open("w") as output:
|
|
# A session of its own lets a timeout or interrupt stop the whole process tree:
|
|
# killing only Cargo would leave its compiler and build-script children running
|
|
# after this script exits and releases the comparison lock.
|
|
process = subprocess.Popen(
|
|
command, cwd=cwd, env=environment, stdout=output,
|
|
stderr=subprocess.STDOUT, start_new_session=True,
|
|
)
|
|
try:
|
|
returncode = process.wait(timeout=timeout)
|
|
except BaseException as error:
|
|
os.killpg(process.pid, signal.SIGKILL)
|
|
process.wait()
|
|
if isinstance(error, subprocess.TimeoutExpired):
|
|
raise RuntimeError(f"{error}; see {log}") from error
|
|
raise
|
|
if returncode != 0:
|
|
raise RuntimeError(f"{command[0]} exited with status {returncode}; see {log}")
|
|
return time.monotonic() - started
|
|
|
|
|
|
def git(root, *arguments):
|
|
return subprocess.check_output(["git", "-C", str(root), *arguments], text=True).strip()
|
|
|
|
|
|
def prepare(root, cache, revision, features, timeout, environment):
|
|
commit = git(root, "rev-parse", "--verify", f"{revision}^{{commit}}")
|
|
source = cache / "sources" / commit
|
|
source.mkdir(parents=True, exist_ok=True)
|
|
if not (source / ".archive-complete").exists():
|
|
archive = cache / f"{commit}.tar"
|
|
with archive.open("wb") as output:
|
|
subprocess.run(["git", "-C", str(root), "archive", commit],
|
|
stdout=output, check=True, timeout=timeout)
|
|
with tarfile.open(archive) as contents:
|
|
contents.extractall(source, filter="data")
|
|
(source / ".archive-complete").touch()
|
|
digest = hashlib.sha256()
|
|
for relative in FIXTURE_FILES:
|
|
content = (root / relative).read_bytes()
|
|
digest.update(str(relative).encode())
|
|
digest.update(content)
|
|
destination = source / relative
|
|
if not destination.exists() or destination.read_bytes() != content:
|
|
destination.write_bytes(content)
|
|
build_key = hashlib.sha256(
|
|
(commit + digest.hexdigest() + features).encode()
|
|
).hexdigest()[:20]
|
|
artifacts = cache / "binaries" / build_key
|
|
artifacts.mkdir(parents=True, exist_ok=True)
|
|
# Cargo's workspace fingerprints use mtimes, not source contents. Archives
|
|
# from different commits can otherwise reuse the preceding revision's binary.
|
|
environment = dict(environment, CARGO_TARGET_DIR=str(cache / "build" / commit))
|
|
feature_log = artifacts / "features.log"
|
|
run(
|
|
["cargo", "tree", "--locked", "-p", "gpui_platform", "--features", features,
|
|
"-e", "normal,build,dev,features"],
|
|
cwd=source, environment=environment, log=feature_log, timeout=timeout,
|
|
)
|
|
if 'feature "test-support"' in feature_log.read_text():
|
|
raise RuntimeError(f"benchmark graph enables test-support: {feature_log}")
|
|
command = [
|
|
"cargo", "bench", "--locked", "-p", "gpui_platform", "--features", features,
|
|
"--profile", "release-fast", "--no-run", "--message-format=json",
|
|
]
|
|
for target in TARGETS:
|
|
command += ["--bench", target]
|
|
elapsed = run(command, cwd=source, environment=environment,
|
|
log=artifacts / "build.log", timeout=timeout)
|
|
binaries = {}
|
|
for line in (artifacts / "build.log").read_text().splitlines():
|
|
try:
|
|
message = json.loads(line)
|
|
except json.JSONDecodeError:
|
|
continue
|
|
if message.get("reason") != "compiler-artifact" or not message.get("executable"):
|
|
continue
|
|
name = message["target"]["name"]
|
|
if name in TARGETS:
|
|
# Cargo may overwrite these paths on the next revision's build.
|
|
destination = artifacts / name
|
|
shutil.copy2(message["executable"], destination)
|
|
binaries[name] = destination
|
|
if set(binaries) != set(TARGETS):
|
|
raise RuntimeError(f"missing benchmark executable; see {artifacts / 'build.log'}")
|
|
binary_hashes = {}
|
|
for target, binary in binaries.items():
|
|
with binary.open("rb") as executable:
|
|
binary_hashes[target] = hashlib.file_digest(executable, "sha256").hexdigest()
|
|
return {
|
|
"revision": revision, "commit": commit, "build_seconds": elapsed,
|
|
"gpui_tree": git(root, "rev-parse", f"{commit}:crates/gpui"),
|
|
"binary_sha256": binary_hashes,
|
|
"benchmark_sha256": digest.hexdigest(),
|
|
"lockfile_sha256": hashlib.sha256((source / "Cargo.lock").read_bytes()).hexdigest(),
|
|
"source": str(source), "binaries": binaries,
|
|
}
|
|
|
|
|
|
def verify_distinct_artifacts(revisions):
|
|
baseline, candidate = revisions
|
|
if baseline["gpui_tree"] != candidate["gpui_tree"]:
|
|
for target in TARGETS:
|
|
if baseline["binary_sha256"][target] == candidate["binary_sha256"][target]:
|
|
raise RuntimeError(
|
|
f"{target} binaries are byte-identical despite different crates/gpui trees; "
|
|
"refusing a potentially contaminated benchmark comparison"
|
|
)
|
|
|
|
|
|
def measure(binary, identifier, directory, environment, timeout, slow):
|
|
directory.mkdir(parents=True, exist_ok=True)
|
|
environment = dict(environment, CRITERION_HOME=str(directory / "criterion"))
|
|
command = [str(binary), "--bench", "^" + re.escape(identifier) + "$", "--noplot"]
|
|
if not slow:
|
|
command += FAST
|
|
elapsed = run(command, cwd=directory, environment=environment,
|
|
log=directory / "run.log", timeout=timeout)
|
|
samples = list((directory / "criterion").glob("**/new/sample.json"))
|
|
if len(samples) != 1:
|
|
raise RuntimeError(f"expected one measured case, got {len(samples)}: {directory}")
|
|
sample = json.loads(samples[0].read_text())
|
|
counts = [total / count for total, count in zip(sample["times"], sample["iters"])]
|
|
wall = re.search(
|
|
r"wall time per iteration: median ([\d.]+) (\S+) \(min ([\d.]+), max ([\d.]+)",
|
|
(directory / "run.log").read_text(),
|
|
)
|
|
if wall is None:
|
|
raise RuntimeError(f"missing wall-time secondary: {directory / 'run.log'}")
|
|
units = {"ns": 1e-9, "µs": 1e-6, "us": 1e-6, "ms": 1e-3, "s": 1}
|
|
return {
|
|
"instructions": statistics.median(counts),
|
|
"sample_min": min(counts), "sample_max": max(counts),
|
|
"wall_seconds": float(wall[1]) * units[wall[2]],
|
|
"elapsed_seconds": elapsed, "iterations": sum(sample["iters"]),
|
|
}
|
|
|
|
|
|
def spread(values):
|
|
return (max(values) - min(values)) / statistics.median(values) * 100
|
|
|
|
|
|
def main():
|
|
started = time.monotonic()
|
|
parser = argparse.ArgumentParser(description=__doc__,
|
|
formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
parser.add_argument("baseline")
|
|
parser.add_argument("candidate")
|
|
parser.add_argument("--filter", help="Criterion regex, applied to all three binaries")
|
|
parser.add_argument("--full", action="store_true")
|
|
parser.add_argument("--slow", action="store_true")
|
|
parser.add_argument("--repetitions", type=int, default=3)
|
|
parser.add_argument("--seed", type=int, default=0)
|
|
# Text benchmarks need the platform's real text system: x11 on Linux, font-kit on macOS.
|
|
parser.add_argument("--features", default={"linux": "bench-support,x11",
|
|
"darwin": "bench-support,font-kit"}.get(
|
|
sys.platform, "bench-support"))
|
|
parser.add_argument("--timeout", type=int, default=300,
|
|
help="hard timeout for EACH build/run, at most 300 seconds")
|
|
parser.add_argument("--cache", type=Path)
|
|
args = parser.parse_args()
|
|
if not 1 <= args.timeout <= 300 or args.repetitions < 1 or args.seed < 0:
|
|
parser.error("timeout must be 1..300, repetitions positive and seed nonnegative")
|
|
if args.filter and args.full:
|
|
parser.error("--filter and --full are mutually exclusive")
|
|
if "bench-support" not in args.features.split(",") or "test-support" in args.features:
|
|
parser.error("features must include bench-support and must not include test-support")
|
|
root = Path(git(Path(__file__).resolve().parent, "rev-parse", "--show-toplevel"))
|
|
cache = (args.cache or root / "target/gpui-bench-compare").resolve()
|
|
cache.mkdir(parents=True, exist_ok=True)
|
|
lock = (cache / "compare.lock").open("w")
|
|
try:
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
except BlockingIOError as error:
|
|
raise RuntimeError(f"another comparison is using {cache}") from error
|
|
output = cache / "runs" / time.strftime("%Y%m%d-%H%M%S")
|
|
output.mkdir(parents=True, exist_ok=False)
|
|
environment = dict(os.environ, BENCH_MEASUREMENT="foreground-instructions",
|
|
SEED=str(args.seed), ITERATIONS="1")
|
|
environment["PATH"] = str(Path.home() / ".cargo/bin") + os.pathsep + environment["PATH"]
|
|
revisions = []
|
|
for revision in [args.baseline, args.candidate]:
|
|
print(f"Building {revision} (once)...", flush=True)
|
|
revisions.append(prepare(root, cache, revision, args.features, args.timeout, environment))
|
|
verify_distinct_artifacts(revisions)
|
|
if revisions[0]["lockfile_sha256"] != revisions[1]["lockfile_sha256"]:
|
|
print("NOTE: revisions have different Cargo.lock files; dependency changes may affect results.")
|
|
cases = {}
|
|
for target in TARGETS:
|
|
listings = []
|
|
pattern = args.filter if args.filter else "" if args.full else CORE[target]
|
|
for index, revision in enumerate(revisions):
|
|
log = output / f"list-{index}-{target}.log"
|
|
run([str(revision["binaries"][target]), "--list", pattern],
|
|
cwd=output, environment=environment, log=log, timeout=args.timeout)
|
|
listings.append([line.removesuffix(": benchmark") for line in log.read_text().splitlines()
|
|
if line.endswith(": benchmark")])
|
|
if listings[0] != listings[1]:
|
|
raise RuntimeError(f"benchmark IDs differ for {target}")
|
|
cases[target] = listings[0]
|
|
if not any(cases.values()):
|
|
raise RuntimeError("filter selected no benchmarks")
|
|
# Verify all selected paths once; exercise Criterion's quick path before sampling.
|
|
for index, revision in enumerate(revisions):
|
|
for target, identifiers in cases.items():
|
|
if not identifiers:
|
|
continue
|
|
binary = revision["binaries"][target]
|
|
pattern = "^(" + "|".join(re.escape(value) for value in identifiers) + ")$"
|
|
for mode, selection in [("--test", pattern),
|
|
("--quick", "^" + re.escape(identifiers[0]) + "$")]:
|
|
run([str(binary), "--bench", selection, mode, "--noplot",
|
|
"--measurement-time", "0.3"], cwd=output, environment=environment,
|
|
log=output / f"smoke-{index}-{target}{mode}.log", timeout=args.timeout)
|
|
results = {}
|
|
print("Measuring sequentially; instructions are foreground retired instructions per frame/operation.",
|
|
flush=True)
|
|
print(f"{'benchmark':76} {'base M':>9} {'new M':>9} {'change':>9} "
|
|
f"{'spread B/N':>14} {'wall ms B/N':>17} {'run s B/N':>13}", flush=True)
|
|
for target, identifiers in cases.items():
|
|
for identifier in identifiers:
|
|
pair = [[], []]
|
|
case_number = len(results)
|
|
for repetition in range(args.repetitions):
|
|
for index in ([0, 1] if repetition % 2 == 0 else [1, 0]):
|
|
revision = revisions[index]
|
|
directory = output / f"{case_number:03}-{repetition}-{index}"
|
|
pair[index].append(measure(revision["binaries"][target], identifier, directory,
|
|
environment, args.timeout, args.slow))
|
|
results[identifier] = pair
|
|
instruction_runs = [[value["instructions"] for value in side] for side in pair]
|
|
baseline, candidate = map(statistics.median, instruction_runs)
|
|
walls = [statistics.median(value["wall_seconds"] for value in side) * 1000 for side in pair]
|
|
durations = [statistics.median(value["elapsed_seconds"] for value in side) for side in pair]
|
|
noise = (f"{spread(instruction_runs[0]):5.2f}/{spread(instruction_runs[1]):5.2f}%"
|
|
if args.repetitions > 1 else " n/a")
|
|
print(f"{identifier:76} {baseline / 1e6:9.3f} {candidate / 1e6:9.3f} "
|
|
f"{(candidate / baseline - 1) * 100:+8.2f}% "
|
|
f"{noise} "
|
|
f"{walls[0]:7.3f}/{walls[1]:7.3f} "
|
|
f"{durations[0]:5.2f}/{durations[1]:5.2f}", flush=True)
|
|
if args.repetitions > 1 and any(spread(values) > 1 for values in instruction_runs):
|
|
print(" NOTE: >1% instruction spread; confirm this case with --slow before judging a change.",
|
|
flush=True)
|
|
sample_ranges = [
|
|
max((value["sample_max"] - value["sample_min"]) / value["instructions"] * 100
|
|
for value in side)
|
|
for side in pair
|
|
]
|
|
if any(value > 1 for value in sample_ranges):
|
|
print(f" NOTE: within-run instruction sample range B/N "
|
|
f"{sample_ranges[0]:.2f}/{sample_ranges[1]:.2f}%; "
|
|
"inspect raw samples for drift/outliers before judging a change.", flush=True)
|
|
summary = {
|
|
"configuration": vars(args) | {"cache": str(cache)},
|
|
"revisions": [{key: value for key, value in revision.items() if key != "binaries"}
|
|
for revision in revisions],
|
|
"results": results,
|
|
"cases": cases,
|
|
"environment": {name: value for name, value in environment.items()
|
|
if name in ("BENCH_MEASUREMENT", "SEED", "ITERATIONS",
|
|
"RUSTFLAGS")},
|
|
}
|
|
(output / "summary.json").write_text(json.dumps(summary, indent=2, default=str) + "\n")
|
|
for index, revision in enumerate(revisions):
|
|
seconds = sum(value["elapsed_seconds"] for pair in results.values() for value in pair[index])
|
|
print(f"{revision['commit'][:12]}: build {revision['build_seconds']:.2f}s, "
|
|
f"measured processes {seconds:.2f}s ({args.repetitions} runs/case)")
|
|
for target, identifiers in cases.items():
|
|
seconds = sum(value["elapsed_seconds"] for identifier in identifiers
|
|
for value in results[identifier][index])
|
|
print(f" {target}: {seconds:.2f}s")
|
|
print(f"Total including builds, discovery, smoke and quick checks: {time.monotonic() - started:.2f}s")
|
|
print(f"Raw samples, frame reports and summary: {output}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
main()
|
|
except (RuntimeError, subprocess.CalledProcessError) as error:
|
|
sys.exit(str(error))
|