diff --git a/.github/workflows/community_pr_board.yml b/.github/workflows/community_pr_board.yml index 82e5df9419f..8f5bd122b22 100644 --- a/.github/workflows/community_pr_board.yml +++ b/.github/workflows/community_pr_board.yml @@ -2,10 +2,11 @@ # # When an area/platform label is added to or already present when reopening a # community PR (not staff, not bot), the PR is added to the project board with a -# Track field set to the matching review area group. Status transitions for -# assignment, re-request, and comment events are handled here. Review-based -# status changes (approved → "In Progress (us)", changes requested → "In -# Progress (author)") are handled by built-in board automations. +# Track field set to the matching review area group, and its signal fields are +# recomputed. Status transitions for assignment, re-request, and comment events +# are handled here. Review-based status changes (approved → "In Progress (us)", +# changes requested → "In Progress (author)") are handled by built-in board +# automations. # # See script/community-pr-track-mapping.json for the label→track mapping. @@ -22,7 +23,7 @@ on: workflow_dispatch: inputs: pr_number: - description: "PR number to process (re-resolves track from current labels)" + description: "PR number to process (re-resolves Track and recomputes signals)" required: true type: number diff --git a/.github/workflows/community_pr_board_refresh.yml b/.github/workflows/community_pr_board_refresh.yml index 81540f3ace9..db88dfb5a8d 100644 --- a/.github/workflows/community_pr_board_refresh.yml +++ b/.github/workflows/community_pr_board_refresh.yml @@ -1,10 +1,10 @@ -# Community PR Board — daily meta information refresh +# Community PR Board — daily signal fields refresh # # Walks every open PR on the community board and recomputes its signal # fields. Backstop for changes that don't reach the event-driven workflow, # either because the relevant webhook isn't subscribed or doesn't fire at all. -name: PR Board Meta Fields Refresh +name: Community PR Board Signals Refresh on: schedule: @@ -34,6 +34,7 @@ jobs: owner: zed-industries repositories: zed permission-issues: read + permission-members: read permission-organization-projects: write permission-pull-requests: read diff --git a/.github/workflows/slack_notify_community_automation_failure.yml b/.github/workflows/slack_notify_community_automation_failure.yml index c1abf7a4986..e26a7e44a8f 100644 --- a/.github/workflows/slack_notify_community_automation_failure.yml +++ b/.github/workflows/slack_notify_community_automation_failure.yml @@ -13,9 +13,9 @@ on: - "Comment on potential duplicate bug/crash reports" - "Track duplicate bot effectiveness" - "Community PR Board" + - "Community PR Board Signals Refresh" - "Community PR Cleanup" - "Maintainer Edits Nudge" - - "PR Board Meta Fields Refresh" - "PR Issue Labeler" - "Welcome First Contribution" - "Guild Assignment Status" diff --git a/script/github-community-pr-board.py b/script/github-community-pr-board.py index 96a12cb7d6d..87b6fe5922a 100644 --- a/script/github-community-pr-board.py +++ b/script/github-community-pr-board.py @@ -1,12 +1,13 @@ #!/usr/bin/env python3 """ Route community PRs to the correct review track on a GitHub Project board, -and surface signal fields (Size, Issue Linked, Contributor, Upvotes) that the -board's views can filter and sort on. +and surface signal fields (Size, Issue Linked, Contributor, Upvotes, Last +push, Merge conflicts, Opened with no self-review) that the board's views +can filter and sort on. Reads the event payload dispatched by the GitHub Actions workflow and: - On `labeled` or `reopened`: adds the PR to the board (idempotent), sets the - Track field to the most specific matching track, and recomputes signal fields. + Track field to the most specific matching track, and recomputes signals. - On `unlabeled`: re-resolves Track from remaining labels, or clears it if no area/platform labels remain (PR stays on the board for visibility). Signals are recomputed if the PR is on the board. @@ -24,17 +25,20 @@ Reads the event payload dispatched by the GitHub Actions workflow and: recomputes signals for a manually specified PR. - When run with REFRESH_ALL=1: walks every open PR on the board and recomputes its signals. Used by the scheduled refresh workflow as a - belt-and-suspenders pass against missed webhook events (notably - intentionally-unsubscribed `synchronize` events that change Size, and - sidebar edits to linked issues that don't fire a PR webhook). + belt-and-suspenders pass against changes that don't reach the + event-driven workflow (notably intentionally-unsubscribed `synchronize` + events that change Size, Last push and Merge conflicts, pushes to main + that change Merge conflicts, reactions, comments and reviews that change + Upvotes, and sidebar edits to linked issues that don't fire a PR webhook). Review-based status changes (approved → "In Progress (us)", changes requested → "In Progress (author)") are handled by built-in board automations, not this script. -Signal fields are written best-effort: if a field or option is missing -from the project, the script logs and skips it rather than failing, so -new fields can be rolled out in the project UI independently of deploys. +Signal fields are written only when their value changed, and best-effort: +if a field or option is missing from the project, the script logs and +skips it rather than failing, so new fields can be rolled out in the +project UI independently of deploys. Requires: requests (pip install requests) @@ -46,20 +50,22 @@ Usage (called by the workflow, not directly): import json import os import sys +import time +from itertools import batched from pathlib import Path -import requests from github_helpers import ( add_github_project_item, fetch_github_project, github_graphql, github_rest_api, + is_staff_member, set_github_project_field, ) REPO_OWNER = "zed-industries" REPO_NAME = "zed" -STAFF_TEAM_SLUG = "staff" +REPO_FULL_NAME = f"{REPO_OWNER}/{REPO_NAME}" SKIP_LABELS = {"staff", "bot"} @@ -71,46 +77,103 @@ STATUS_IN_PROGRESS_AUTHOR = ( MAPPING_PATH = Path(__file__).parent / "community-pr-track-mapping.json" +MERGE_CONFLICTS_FIELD = "Merge conflicts" +HAS_CONFLICTS_OPTION = "❌" +NO_SELF_REVIEW_LABEL = "no self-review" +NO_SELF_REVIEW_OPTION = "AI?" +POSITIVE_REACTIONS = {"THUMBS_UP", "HEART", "HOORAY", "ROCKET"} +BOTS_WITH_USER_ACCOUNTS = {"zed-industries-bot"} +# GitHub answers 502 to batches of 25. +BOARD_ITEMS_PER_QUERY = 10 +MAX_NODE_IDS_PER_QUERY = 100 +# GitHub forgets mergeability whenever main moves, and only computes it again +# once asked, so a recheck asks, waits, and reads again. +MERGEABILITY_RECHECK_DELAY_SECONDS = 15 +MERGEABILITY_RECHECK_ROUNDS = 3 + +CONNECTION_NODE_SELECTIONS = { + "reactions": "content user { login }", + "comments": "author { __typename login }", + "reviews": "author { __typename login }", +} +# %-formatting, since every GraphQL brace would need doubling in an f-string. +BOARD_ITEM_FRAGMENT = """ +fragment BoardItem on ProjectV2Item { + id + isArchived + fieldValues(first: 30) { + nodes { + ... on ProjectV2ItemFieldSingleSelectValue { name field { ... on ProjectV2FieldCommon { name } } } + ... on ProjectV2ItemFieldNumberValue { number field { ... on ProjectV2FieldCommon { name } } } + ... on ProjectV2ItemFieldDateValue { date field { ... on ProjectV2FieldCommon { name } } } + } + } + content { + ... on PullRequest { + id + number + state + additions + deletions + mergeable + author { login } + labels(first: 50) { nodes { name } } + commits(last: 1) { nodes { commit { committedDate } } } + reactions(first: 100) { nodes { %(reactions)s } pageInfo { hasNextPage endCursor } } + comments(first: 100) { nodes { %(comments)s } pageInfo { hasNextPage endCursor } } + reviews(first: 100) { nodes { %(reviews)s } pageInfo { hasNextPage endCursor } } + closingIssuesReferences(first: 20) { + nodes { + __typename + id + repository { nameWithOwner } + issueType { name } + reactions(first: 100) { nodes { %(reactions)s } pageInfo { hasNextPage endCursor } } + comments(first: 100) { nodes { %(comments)s } pageInfo { hasNextPage endCursor } } + } + } + } + } +} +""" % CONNECTION_NODE_SELECTIONS + def sync_track_from_labels(pr, project_number): """Sync the PR's Track field on the board with its current labels. - Returns (project, project_item) if the PR ends up on (or stays on) - the board, so callers can do further per-item work without a second - project/item lookup. Returns (None, None) otherwise. + Returns (project, item) if the PR ends up on (or stays on) the board, so + callers can refresh its signals without fetching it again. Returns + (None, None) otherwise. """ pr_labels = {label["name"] for label in pr.get("labels", [])} track_name = resolve_track(pr_labels, load_mapping()) project = fetch_github_project(project_number) - project_item = github_find_project_item(project["id"], pr["node_id"]) + item_id = github_find_project_item_id(project["id"], pr["node_id"]) + if not item_id: + if not track_name: + print( + f"No track matched for labels on PR #{pr['number']}, not on board, nothing to do" + ) + return None, None + item_id = add_github_project_item(project["id"], pr["node_id"]) - if not track_name: - if project_item: - github_clear_field(project, project_item, "Track") - print(f"No track matched, cleared Track on PR #{pr['number']}") - return project, project_item - print( - f"No track matched for labels on PR #{pr['number']}, not on board, nothing to do" - ) - return None, None - - print(f"Resolved track: {track_name}") - - if not project_item: - project_item = add_github_project_item(project["id"], pr["node_id"]) - set_github_project_field(project, project_item, "Track", track_name) - return project, project_item + [item] = github_fetch_board_items([item_id]) + current_track = current_field_values(item).get("Track") + # Case-insensitive like the write, since the mapping is maintained by hand. + if (current_track or "").casefold() != (track_name or "").casefold(): + set_github_project_field(project, item_id, "Track", track_name) + return project, item def set_progress_status_on_assignment(pr, assignee_login, project_number): """Set Status to 'In Progress (us)' when a staff member self-assigns.""" - if not github_is_staff_member(assignee_login): + if not is_staff_member(assignee_login): print(f"Assignee '{assignee_login}' is not a staff member, skipping") return project = fetch_github_project(project_number) - item_id = github_find_project_item(project["id"], pr["node_id"]) + item_id = github_find_project_item_id(project["id"], pr["node_id"]) if not item_id: print(f"PR #{pr['number']} not on board, skipping assignment status update") return @@ -125,7 +188,7 @@ def return_to_reviewer(pr, project_number, reason): by re-requesting review or by commenting on the PR. """ project = fetch_github_project(project_number) - item_id = github_find_project_item(project["id"], pr["node_id"]) + item_id = github_find_project_item_id(project["id"], pr["node_id"]) if not item_id: print(f"PR #{pr['number']} not on board, skipping") return @@ -174,42 +237,207 @@ def compute_contributor(pr_labels): return "Returning" -def recompute_signals(pr, project, project_item): - """Recompute Size, Contributor, Issue Linked, and Upvotes on a board item. +def closing_issues_in_repository(pr): + return [ + issue + for issue in pr["closingIssuesReferences"]["nodes"] + if issue["repository"]["nameWithOwner"] == REPO_FULL_NAME + ] - Best-effort: missing fields/options are logged and skipped so the - project's field set can be rolled out independently. - """ - pr_labels = {label["name"] for label in pr.get("labels", [])} - total_changes = pr.get("additions", 0) + pr.get("deletions", 0) - author_login = (pr.get("user") or {}).get("login") - issue_type, upvotes = github_pr_issue_signals(pr["node_id"], author_login) +def linked_issue_type(pr): + """Return the most significant type among the repository's issues the PR closes.""" + closing_issues = closing_issues_in_repository(pr) + if not closing_issues: + return "No issue" - set_field_optional(project, project_item, "Size", compute_size_bucket(total_changes)) - set_field_optional(project, project_item, "Contributor", compute_contributor(pr_labels)) - set_field_optional(project, project_item, "Issue Linked", issue_type) - set_number_field_optional(project, project_item, "Upvotes", upvotes) + type_names = { + issue["issueType"]["name"] + for issue in closing_issues + if issue.get("issueType") + } + for type_name in ("Crash", "Bug", "Feature", "Docs"): + if type_name in type_names: + return type_name + + print( + f"Warning: PR has {len(closing_issues)} linked issue(s) but none with " + f"a recognized type (saw: {type_names or 'no type set'})" + ) + return "No issue" + + +def github_connection_page(subject, connection_name, cursor): + data = github_graphql( + f""" + query($id: ID!, $cursor: String!) {{ + node(id: $id) {{ + ... on {subject['__typename']} {{ + page: {connection_name}(first: 100, after: $cursor) {{ + nodes {{ {CONNECTION_NODE_SELECTIONS[connection_name]} }} + pageInfo {{ hasNextPage endCursor }} + }} + }} + }} + }} + """, + {"id": subject["id"], "cursor": cursor}, + ) + return data["node"]["page"] + + +def connection_nodes(subject, connection_name): + connection = subject[connection_name] + while True: + yield from connection["nodes"] + if not connection["pageInfo"]["hasNextPage"]: + return + connection = github_connection_page( + subject, connection_name, connection["pageInfo"]["endCursor"] + ) + + +def engaged_users(pr): + for subject in [pr, *closing_issues_in_repository(pr)]: + for reaction in connection_nodes(subject, "reactions"): + if reaction["content"] in POSITIVE_REACTIONS: + yield reaction["user"] + for comment in connection_nodes(subject, "comments"): + yield comment["author"] + for review in connection_nodes(pr, "reviews"): + yield review["author"] + + +def is_bot(user): + login = user["login"].lower() + return ( + user.get("__typename") == "Bot" + or login.endswith("[bot]") + or login in BOTS_WITH_USER_ACCOUNTS + ) + + +def count_interested_people(pr): + author_login = (pr["author"] or {}).get("login") + interested_logins = { + user["login"] + for user in engaged_users(pr) + if user + and user["login"] != author_login + and not is_bot(user) + and not is_staff_member(user["login"]) + } + return len(interested_logins) + + +def merge_conflicts_value(mergeable): + return HAS_CONFLICTS_OPTION if mergeable == "CONFLICTING" else None + + +def compute_signals(pr): + pr_labels = {label["name"] for label in pr["labels"]["nodes"]} + signals = { + "Size": compute_size_bucket(pr["additions"] + pr["deletions"]), + "Contributor": compute_contributor(pr_labels), + "Issue Linked": linked_issue_type(pr), + # Counts people, not reactions, despite the field's name. + "Upvotes": count_interested_people(pr), + # The committer date moves with rebases and merges from main. + "Last push": pr["commits"]["nodes"][0]["commit"]["committedDate"][:10], + "Opened with no self-review": ( + NO_SELF_REVIEW_OPTION if NO_SELF_REVIEW_LABEL in pr_labels else None + ), + } + # Left out while GitHub is still computing mergeability, so the current value stays. + if pr["mergeable"] != "UNKNOWN": + signals[MERGE_CONFLICTS_FIELD] = merge_conflicts_value(pr["mergeable"]) + return signals + + +def current_field_values(item): + return { + node["field"]["name"]: node.get("name", node.get("number", node.get("date"))) + for node in item["fieldValues"]["nodes"] + if "field" in node + } + + +def write_changed_fields(project, item_id, values, current_values): + for field_name, value in values.items(): + if current_values.get(field_name) != value: + set_github_project_field( + project, item_id, field_name, value, missing_ok=True + ) + + +def refresh_signals(project, item): + pr = item["content"] + print(f"--- Refreshing signals on PR #{pr['number']} ---") + write_changed_fields( + project, item["id"], compute_signals(pr), current_field_values(item) + ) def refresh_signals_if_on_board(pr, project_number): - """Recompute signals for a PR only if it's already on the board.""" project = fetch_github_project(project_number) - project_item = github_find_project_item(project["id"], pr["node_id"]) - if not project_item: + item_id = github_find_project_item_id(project["id"], pr["node_id"]) + if not item_id: print(f"PR #{pr['number']} not on board, skipping signal refresh") return - recompute_signals(pr, project, project_item) + [item] = github_fetch_board_items([item_id]) + refresh_signals(project, item) + + +def recheck_mergeability(project, mergeability_rechecks): + """Write Merge conflicts on items whose mergeability was still UNKNOWN on + the first read. Returns the number of items that failed. + """ + failed = 0 + pending_rechecks = mergeability_rechecks + for _ in range(MERGEABILITY_RECHECK_ROUNDS): + time.sleep(MERGEABILITY_RECHECK_DELAY_SECONDS) + still_unknown = [] + for batch in batched(pending_rechecks, MAX_NODE_IDS_PER_QUERY): + try: + mergeable_by_pr_id = github_fetch_mergeable_by_pr_id( + [recheck["pr_id"] for recheck in batch] + ) + except Exception as error: + print(f"Failed to re-read mergeability of {len(batch)} PR(s): {error}") + failed += len(batch) + continue + for recheck in batch: + mergeable = mergeable_by_pr_id.get(recheck["pr_id"], "UNKNOWN") + if mergeable == "UNKNOWN": + still_unknown.append(recheck) + continue + try: + write_changed_fields( + project, + recheck["item_id"], + {MERGE_CONFLICTS_FIELD: merge_conflicts_value(mergeable)}, + {MERGE_CONFLICTS_FIELD: recheck["merge_conflicts"]}, + ) + except Exception as error: + print(f"Failed to write Merge conflicts on PR #{recheck['number']}: {error}") + failed += 1 + pending_rechecks = still_unknown + if not pending_rechecks: + break + if pending_rechecks: + print(f"Mergeability still unknown for {len(pending_rechecks)} PR(s), left as they were") + return failed def refresh_all_board_items(project_number): - """Walk every open PR on the board and recompute its signals. + """Walk every open PR on the board and refresh its signals. - Backstop for the daily refresh workflow. Individual PR failures are - logged and don't abort the rest of the run. + Backstop for the daily refresh workflow. A failure on one PR doesn't + stop the others, but fails the run at the end. """ project = fetch_github_project(project_number) - processed = skipped = errors = 0 + listed_items = [] + skipped = 0 for item in github_list_project_items(project["id"]): if item.get("isArchived"): skipped += 1 @@ -221,28 +449,51 @@ def refresh_all_board_items(project_number): if content.get("state") != "OPEN" or content.get("isDraft"): skipped += 1 continue - label_names = [n["name"] for n in content["labels"]["nodes"]] + label_names = [label["name"] for label in content["labels"]["nodes"]] if set(label_names) & SKIP_LABELS: skipped += 1 continue - pr = { - "number": content["number"], - "node_id": content["id"], - "labels": [{"name": n} for n in label_names], - "additions": content["additions"], - "deletions": content["deletions"], - "user": {"login": (content.get("author") or {}).get("login")}, - } - print(f"--- Refreshing PR #{pr['number']} ---") + listed_items.append(item) + + processed = failed = 0 + mergeability_rechecks = [] + for batch in batched(listed_items, BOARD_ITEMS_PER_QUERY): try: - recompute_signals(pr, project, item["id"]) - processed += 1 - except Exception as exc: - print(f"Failed to refresh PR #{pr['number']}: {exc}") - errors += 1 + items = github_fetch_board_items([item["id"] for item in batch]) + except Exception as error: + # One broken item fails the whole query, so they're fetched one by one. + print(f"Failed to fetch {len(batch)} items at once, retrying one by one: {error}") + items = [None] * len(batch) + for listed_item, item in zip(batch, items): + try: + if item is None: + [item] = github_fetch_board_items([listed_item["id"]]) + pr = item["content"] + # Closed since the board was listed, and archived along with it. + if item["isArchived"] or pr["state"] != "OPEN": + skipped += 1 + continue + refresh_signals(project, item) + processed += 1 + if pr["mergeable"] == "UNKNOWN": + mergeability_rechecks.append({ + "item_id": item["id"], + "pr_id": pr["id"], + "number": pr["number"], + "merge_conflicts": current_field_values(item).get( + MERGE_CONFLICTS_FIELD + ), + }) + except Exception as error: + print(f"Failed to refresh PR #{listed_item['content']['number']}: {error}") + failed += 1 + if mergeability_rechecks: + failed += recheck_mergeability(project, mergeability_rechecks) print( - f"Refresh complete: {processed} processed, {skipped} skipped, {errors} errors" + f"Refresh complete: {processed} processed, {skipped} skipped, {failed} failed" ) + if failed: + raise RuntimeError(f"Failed to refresh {failed} PR(s), see above") def load_mapping(path=MAPPING_PATH): @@ -263,29 +514,11 @@ def resolve_track(pr_labels, tracks): return None -def github_is_staff_member(username): - """Check if a user is a member of the staff team.""" - try: - github_rest_api( - "GET", f"orgs/{REPO_OWNER}/teams/{STAFF_TEAM_SLUG}/members/{username}" - ) - return True - except requests.HTTPError as error: - if error.response.status_code == 404: - return False - print(f"Warning: failed to check staff membership for '{username}': {error}") - return False - except requests.RequestException as error: - print(f"Warning: failed to check staff membership for '{username}': {error}") - return False - - def github_fetch_pr(pr_number): """Fetch a PR by number via the REST API.""" return github_rest_api("GET", f"repos/{REPO_OWNER}/{REPO_NAME}/pulls/{pr_number}") - def github_list_project_items(project_id): """Yield every item on a project board, paginating as needed. @@ -307,13 +540,9 @@ def github_list_project_items(project_id): content { __typename ... on PullRequest { - id number state isDraft - additions - deletions - author { login } labels(first: 50) { nodes { name } } } } @@ -333,123 +562,34 @@ def github_list_project_items(project_id): cursor = page["pageInfo"]["endCursor"] -POSITIVE_REACTIONS = ( - ("thumbsUp", "THUMBS_UP"), - ("heart", "HEART"), - ("hooray", "HOORAY"), - ("rocket", "ROCKET"), -) - - -def github_reaction_page(subject_node_id, content, cursor): +def github_fetch_board_items(item_ids): data = github_graphql( - """ - query($subjectId: ID!, $content: ReactionContent!, $cursor: String!) { - node(id: $subjectId) { - ... on Reactable { - reactions(first: 100, content: $content, after: $cursor) { - nodes { user { login } } - pageInfo { hasNextPage endCursor } - } - } - } + BOARD_ITEM_FRAGMENT + + """ + query($ids: [ID!]!) { + nodes(ids: $ids) { ...BoardItem } } """, - {"subjectId": subject_node_id, "content": content, "cursor": cursor}, + {"ids": item_ids}, ) - return data["node"]["reactions"] + return data["nodes"] -def add_positive_reactors(subject, author_login, reactors): - for connection_name, content in POSITIVE_REACTIONS: - reactions = subject[connection_name] - while True: - for reaction in reactions["nodes"]: - user = reaction.get("user") - login = user.get("login") if user else None - if login and login != author_login: - reactors.add(login) - - page_info = reactions["pageInfo"] - if not page_info["hasNextPage"]: - break - reactions = github_reaction_page( - subject["id"], content, page_info["endCursor"] - ) - - -def github_pr_issue_signals(pr_node_id, author_login): - """Return linked issue type and deduplicated upvotes for a PR. - - Includes reactions on closing issues and excludes the PR author. - """ +def github_fetch_mergeable_by_pr_id(pr_ids): data = github_graphql( """ - fragment PositiveReactions on Reactable { - id - thumbsUp: reactions(first: 100, content: THUMBS_UP) { - nodes { user { login } } - pageInfo { hasNextPage endCursor } - } - heart: reactions(first: 100, content: HEART) { - nodes { user { login } } - pageInfo { hasNextPage endCursor } - } - hooray: reactions(first: 100, content: HOORAY) { - nodes { user { login } } - pageInfo { hasNextPage endCursor } - } - rocket: reactions(first: 100, content: ROCKET) { - nodes { user { login } } - pageInfo { hasNextPage endCursor } - } - } - - query($prId: ID!) { - node(id: $prId) { - ... on PullRequest { - ...PositiveReactions - closingIssuesReferences(first: 20) { - totalCount - nodes { - issueType { name } - ...PositiveReactions - } - } - } - } + query($ids: [ID!]!) { + nodes(ids: $ids) { ... on PullRequest { id mergeable } } } """, - {"prId": pr_node_id}, + {"ids": pr_ids}, + partial_errors_ok=True, ) - pr = data["node"] - refs = pr["closingIssuesReferences"] - - reactors = set() - add_positive_reactors(pr, author_login, reactors) - for issue in refs["nodes"]: - add_positive_reactors(issue, author_login, reactors) - - if refs["totalCount"] == 0: - return "No issue", len(reactors) - - type_names = { - issue["issueType"]["name"] - for issue in refs["nodes"] - if issue.get("issueType") - } - for priority in ("Crash", "Bug", "Feature", "Docs"): - if priority in type_names: - return priority, len(reactors) - - print( - f"Warning: PR has {refs['totalCount']} linked issue(s) but none with " - f"a recognized type (saw: {type_names or 'no type set'})" - ) - return "No issue", len(reactors) + # PRs GitHub can no longer return come back as null. + return {node["id"]: node["mergeable"] for node in data["nodes"] if node} -def github_find_project_item(project_id, content_node_id): +def github_find_project_item_id(project_id, content_node_id): """Find a PR's item ID on the project board, or None if not present. Uses a read-only query so it won't add the PR as a side effect. @@ -477,127 +617,6 @@ def github_find_project_item(project_id, content_node_id): return None - -def set_field_optional(project, item_id, field_name, option_name): - """Set a single-select field, logging and skipping if the field or - option doesn't exist on the project yet. - - Used for signal fields so they can be added in the project UI - independently of script deploys without crashing the workflow. - """ - field = None - for f in project["fields"]["nodes"]: - if f.get("name") == field_name: - field = f - break - if not field: - print(f"Field '{field_name}' not on project, skipping") - return - option_id = None - for opt in field.get("options", []): - if opt["name"] == option_name: - option_id = opt["id"] - break - if not option_id: - available = [opt["name"] for opt in field.get("options", [])] - print( - f"Option '{option_name}' not in field '{field_name}' " - f"(available: {available}), skipping" - ) - return - github_graphql( - """ - mutation($projectId: ID!, $itemId: ID!, $fieldId: ID!, $optionId: String!) { - updateProjectV2ItemFieldValue(input: { - projectId: $projectId - itemId: $itemId - fieldId: $fieldId - value: { singleSelectOptionId: $optionId } - }) { - projectV2Item { id } - } - } - """, - { - "projectId": project["id"], - "itemId": item_id, - "fieldId": field["id"], - "optionId": option_id, - }, - ) - print(f"Set '{field_name}' to '{option_name}'") - - -def set_number_field_optional(project, item_id, field_name, value): - """Set a number field, logging and skipping if the field doesn't exist - on the project yet. Mirror of `set_field_optional` for non-select fields. - """ - field = None - for f in project["fields"]["nodes"]: - if f.get("name") == field_name: - field = f - break - if not field: - print(f"Field '{field_name}' not on project, skipping") - return - github_graphql( - """ - mutation($projectId: ID!, $itemId: ID!, $fieldId: ID!, $value: Float!) { - updateProjectV2ItemFieldValue(input: { - projectId: $projectId - itemId: $itemId - fieldId: $fieldId - value: { number: $value } - }) { - projectV2Item { id } - } - } - """, - { - "projectId": project["id"], - "itemId": item_id, - "fieldId": field["id"], - "value": value, - }, - ) - print(f"Set '{field_name}' to {value}") - - -def github_clear_field(project, item_id, field_name): - """Clear a single-select field on a project item.""" - field_id = None - for field in project["fields"]["nodes"]: - if field.get("name") == field_name: - field_id = field["id"] - break - - if not field_id: - available = [f["name"] for f in project["fields"]["nodes"] if "name" in f] - raise RuntimeError( - f"Field '{field_name}' not found on project. Available: {available}" - ) - - github_graphql( - """ - mutation($projectId: ID!, $itemId: ID!, $fieldId: ID!) { - clearProjectV2ItemFieldValue(input: { - projectId: $projectId - itemId: $itemId - fieldId: $fieldId - }) { - projectV2Item { id } - } - } - """, - { - "projectId": project["id"], - "itemId": item_id, - "fieldId": field_id, - }, - ) - print(f"Cleared '{field_name}'") - - def github_get_field_value(item_id, field_name): """Read the current value of a single-select field on a project item.""" data = github_graphql( @@ -682,9 +701,9 @@ if __name__ == "__main__": print(f"Processing PR #{pr['number']}: action={action}") if action in ("labeled", "unlabeled", "reopened"): - project, project_item = sync_track_from_labels(pr, project_number) - if project_item: - recompute_signals(pr, project, project_item) + project, item = sync_track_from_labels(pr, project_number) + if item: + refresh_signals(project, item) elif action == "assigned": assignee_login = event.get("assignee", {}).get("login") if not assignee_login: diff --git a/script/github-pr-cleanup.py b/script/github-pr-cleanup.py index 7551ec61c9d..e8ead0aa061 100644 --- a/script/github-pr-cleanup.py +++ b/script/github-pr-cleanup.py @@ -387,8 +387,6 @@ def staff_change_request_at(summary): """Return when staff requested changes, if that's still the latest staff verdict on the pull request.""" reviews = summary["reviews"]["nodes"] - # Most pull requests have no change request at all, and ruling them out - # first saves the staff membership lookups below. if not any(review["state"] == "CHANGES_REQUESTED" for review in reviews): return None for review in reversed(reviews): diff --git a/script/github_helpers.py b/script/github_helpers.py index 166ded78159..baeeed292db 100644 --- a/script/github_helpers.py +++ b/script/github_helpers.py @@ -6,10 +6,16 @@ from functools import cache import requests GITHUB_API_URL = "https://api.github.com" -RETRYABLE_STATUS_CODES = {429, 500, 502, 503, 504} +RETRYABLE_STATUS_CODES = {500, 502, 503, 504} +# GitHub asks to wait at least a minute after a secondary rate limit that +# doesn't say how long. +RATE_LIMIT_DEFAULT_DELAY_SECONDS = 60 MAX_ATTEMPTS = 4 REQUEST_TIMEOUT_SECONDS = 30 NON_COMMUNITY_LABELS = {"staff", "bot"} +PROJECT_FIELD_VALUE_KEYS = {"NUMBER": "number", "DATE": "date", "TEXT": "text"} + +_session = requests.Session() def _github_headers(): @@ -23,6 +29,17 @@ def _github_headers(): } +def _is_rate_limited(response): + return response.status_code == 429 or ( + response.status_code == 403 + and ( + "Retry-After" in response.headers + or response.headers.get("X-RateLimit-Remaining") == "0" + or "rate limit" in response.text.lower() + ) + ) + + def _retry_delay(response, attempt): retry_after = response.headers.get("Retry-After") if retry_after: @@ -30,15 +47,20 @@ def _retry_delay(response, attempt): return max(float(retry_after), 0) except ValueError: pass + reset_at = response.headers.get("X-RateLimit-Reset") + if response.headers.get("X-RateLimit-Remaining") == "0" and reset_at: + return max(float(reset_at) - time.time(), 0) + if _is_rate_limited(response): + return RATE_LIMIT_DEFAULT_DELAY_SECONDS return 2**attempt def _github_request(method, path, *, params=None, body=None, retry=True): url = f"{GITHUB_API_URL}/{path}" - max_attempts = MAX_ATTEMPTS if retry else 1 - for attempt in range(max_attempts): + for attempt in range(MAX_ATTEMPTS): + is_last_attempt = attempt == MAX_ATTEMPTS - 1 try: - response = requests.request( + response = _session.request( method, url, headers=_github_headers(), @@ -47,7 +69,7 @@ def _github_request(method, path, *, params=None, body=None, retry=True): timeout=REQUEST_TIMEOUT_SECONDS, ) except (requests.ConnectionError, requests.Timeout) as error: - if attempt == max_attempts - 1: + if not retry or is_last_attempt: raise delay = 2**attempt print( @@ -57,9 +79,11 @@ def _github_request(method, path, *, params=None, body=None, retry=True): time.sleep(delay) continue - if ( - response.status_code in RETRYABLE_STATUS_CODES - and attempt < max_attempts - 1 + # GitHub doesn't carry out a rate-limited request, so even a write is + # safe to send again. + if not is_last_attempt and ( + _is_rate_limited(response) + or (retry and response.status_code in RETRYABLE_STATUS_CODES) ): delay = _retry_delay(response, attempt) print( @@ -110,16 +134,13 @@ def post_github_comment(owner, repository, issue_number, body): @cache +def _fetch_staff_logins(): + members = github_rest_get_paginated("orgs/zed-industries/teams/staff/members") + return frozenset(member["login"].lower() for member in members) + + def is_staff_member(login): - try: - membership = github_rest_api( - "GET", f"orgs/zed-industries/teams/staff/memberships/{login}" - ) - except requests.HTTPError as error: - if error.response.status_code == 404: - return False - raise - return membership["state"] == "active" + return login.lower() in _fetch_staff_logins() def is_community_pull_request(label_names): @@ -127,9 +148,10 @@ def is_community_pull_request(label_names): return not NON_COMMUNITY_LABELS & {name.lower() for name in label_names} -def github_graphql(query, variables=None, *, partial_errors_ok=False): - # GraphQL uses POST for reads and writes, so classify the operation before retrying. - retry = not query.lstrip().startswith("mutation") +def github_graphql(query, variables=None, *, partial_errors_ok=False, idempotent=False): + # GraphQL uses POST for reads and writes, so classify the operation before + # retrying. `idempotent` marks mutations that are safe to send twice. + retry = idempotent or not query.lstrip().startswith("mutation") response = _github_request( "POST", "graphql", @@ -179,57 +201,90 @@ def add_github_project_item(project_id, content_id): } """, {"projectId": project_id, "contentId": content_id}, + idempotent=True, ) return data["addProjectV2ItemById"]["item"]["id"] -def set_github_project_field(project, item_id, field_name, option_name): - field = next( - (field for field in project["fields"]["nodes"] if field.get("name") == field_name), - None, - ) - if not field: - available = [ - field["name"] for field in project["fields"]["nodes"] if "name" in field - ] - raise RuntimeError( - f"Field '{field_name}' not found on project. Available: {available}" - ) +def _raise_unless_missing_ok(message, missing_ok): + if not missing_ok: + raise RuntimeError(message) + print(f"{message}, skipping") - option_id = next( - ( - option["id"] - for option in field.get("options", []) - # Case-insensitive mainly for Track, which has a mapping maintained manually - if option["name"].casefold() == option_name.casefold() - ), - None, - ) - if not option_id: - available = [option["name"] for option in field.get("options", [])] - raise RuntimeError( - f"Option '{option_name}' not found in field '{field_name}'. " - f"Available: {available}" + +def set_github_project_field(project, item_id, field_name, value, *, missing_ok=False): + """Set a single-select, number, date or text field on a project item, or + clear it when `value` is `None`. + + With `missing_ok`, a missing field or option is logged and skipped instead + of raising. + """ + fields = [field for field in project["fields"]["nodes"] if "name" in field] + field = next((field for field in fields if field["name"] == field_name), None) + if not field: + available = [field["name"] for field in fields] + _raise_unless_missing_ok( + f"Field '{field_name}' not found on project. Available: {available}", + missing_ok, ) + return + + ids = {"projectId": project["id"], "itemId": item_id, "fieldId": field["id"]} + if value is None: + github_graphql( + """ + mutation($projectId: ID!, $itemId: ID!, $fieldId: ID!) { + clearProjectV2ItemFieldValue(input: { + projectId: $projectId + itemId: $itemId + fieldId: $fieldId + }) { + projectV2Item { id } + } + } + """, + ids, + idempotent=True, + ) + print(f"Cleared '{field_name}'") + return + + if "options" in field: + option_id = next( + ( + option["id"] + for option in field["options"] + # Case-insensitive mainly for Track, which has a mapping maintained manually + if option["name"].casefold() == value.casefold() + ), + None, + ) + if not option_id: + available = [option["name"] for option in field["options"]] + _raise_unless_missing_ok( + f"Option '{value}' not found in field '{field_name}'. " + f"Available: {available}", + missing_ok, + ) + return + field_value = {"singleSelectOptionId": option_id} + else: + field_value = {PROJECT_FIELD_VALUE_KEYS[field["dataType"]]: value} github_graphql( """ - mutation($projectId: ID!, $itemId: ID!, $fieldId: ID!, $optionId: String!) { + mutation($projectId: ID!, $itemId: ID!, $fieldId: ID!, $value: ProjectV2FieldValue!) { updateProjectV2ItemFieldValue(input: { projectId: $projectId itemId: $itemId fieldId: $fieldId - value: { singleSelectOptionId: $optionId } + value: $value }) { projectV2Item { id } } } """, - { - "projectId": project["id"], - "itemId": item_id, - "fieldId": field["id"], - "optionId": option_id, - }, + {**ids, "value": field_value}, + idempotent=True, ) - print(f"Set '{field_name}' to '{option_name}'") + print(f"Set '{field_name}'")