| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import http.client |
| 5 | import json |
| 6 | import os |
| 7 | import random |
| 8 | import sys |
| 9 | import time |
| 10 | import urllib.error |
| 11 | import urllib.parse |
| 12 | import urllib.request |
| 13 | from collections.abc import Callable, Mapping, Sequence |
| 14 | from typing import Any |
| 15 | |
| 16 | COMMAND_RUN_CI = "/ci run" |
| 17 | COMMAND_RUN_CI_ALL = "/ci run all" |
| 18 | COMMAND_RUN_CI_NIGHTLY = "/ci run nightly" |
| 19 | COMMAND_RETRY_FAILED = "/ci retry" |
| 20 | COMMAND_CANCEL_CI = "/ci cancel" |
| 21 | COMMAND_RUN_AMD_CI = "/amd-ci run" |
| 22 | COMMAND_RUN_AMD_CI_ALL = "/amd-ci run all" |
| 23 | COMMAND_RUN_AMD_CI_NIGHTLY = "/amd-ci run nightly" |
| 24 | COMMAND_RETRY_AMD_FAILED = "/amd-ci retry" |
| 25 | COMMAND_CANCEL_AMD_CI = "/amd-ci cancel" |
| 26 | ALLOW_STALE_SUFFIX = " --allow-stale" |
| 27 | RUN_CI_COMMAND_ENV = { |
| 28 | COMMAND_RUN_CI: {}, |
| 29 | COMMAND_RUN_CI_ALL: {"RUN_ALL": "1"}, |
| 30 | COMMAND_RUN_CI_NIGHTLY: {"RUN_ALL": "1", "NIGHTLY": "1"}, |
| 31 | COMMAND_RUN_AMD_CI: {}, |
| 32 | COMMAND_RUN_AMD_CI_ALL: {"RUN_ALL": "1"}, |
| 33 | COMMAND_RUN_AMD_CI_NIGHTLY: {"RUN_ALL": "1", "NIGHTLY": "1"}, |
| 34 | } |
| 35 | ALLOW_STALE_COMMANDS = frozenset( |
| 36 | f"{command}{ALLOW_STALE_SUFFIX}" for command in RUN_CI_COMMAND_ENV |
| 37 | ) |
| 38 | RUN_CI_COMMAND_ENV.update( |
| 39 | { |
| 40 | command: RUN_CI_COMMAND_ENV[command.removesuffix(ALLOW_STALE_SUFFIX)] |
| 41 | for command in ALLOW_STALE_COMMANDS |
| 42 | } |
| 43 | ) |
| 44 | UPSTREAM_CI_COMMANDS = frozenset( |
| 45 | { |
| 46 | *(command for command in RUN_CI_COMMAND_ENV if command.startswith("/ci ")), |
| 47 | COMMAND_RETRY_FAILED, |
| 48 | COMMAND_CANCEL_CI, |
| 49 | } |
| 50 | ) |
| 51 | AMD_CI_COMMANDS = frozenset( |
| 52 | { |
| 53 | *(command for command in RUN_CI_COMMAND_ENV if command.startswith("/amd-ci ")), |
| 54 | COMMAND_RETRY_AMD_FAILED, |
| 55 | COMMAND_CANCEL_AMD_CI, |
| 56 | } |
| 57 | ) |
| 58 | RETRY_COMMANDS = frozenset({COMMAND_RETRY_FAILED, COMMAND_RETRY_AMD_FAILED}) |
| 59 | CANCEL_COMMANDS = frozenset({COMMAND_CANCEL_CI, COMMAND_CANCEL_AMD_CI}) |
| 60 | ALL_CI_COMMANDS = UPSTREAM_CI_COMMANDS | AMD_CI_COMMANDS |
| 61 | CI_AUTHORIZED_COMMENT_MARKER = "<!-- vllm-ci-authorized -->" |
| 62 | READY_LABELS = {"ready", "ready-run-all-tests"} |
| 63 | TRUSTED_PERMISSIONS = {"admin", "maintain", "write"} |
| 64 | ACTIVE_BUILD_STATES = { |
| 65 | "blocked", |
| 66 | "creating", |
| 67 | "scheduled", |
| 68 | "running", |
| 69 | "failing", |
| 70 | "canceling", |
| 71 | "waiting", |
| 72 | "waiting_failed", |
| 73 | } |
| 74 | RETRY_STATES = "failed,timed_out,expired" |
| 75 | CANCELABLE_BUILD_STATES = ("scheduled", "running", "failing") |
| 76 | SETUP_STEP_KEYS = { |
| 77 | "ensure-ci-base-amd", |
| 78 | "pre-commit", |
| 79 | "refresh-rocm-base-amd", |
| 80 | } |
| 81 | |
| 82 | |
| 83 | class ApiError(RuntimeError): |
| 84 | def __init__(self, status: int | None, message: str) -> None: |
| 85 | super().__init__(message) |
| 86 | self.status = status |
| 87 | |
| 88 | |
| 89 | class CiPreparationError(RuntimeError): |
| 90 | pass |
| 91 | |
| 92 | |
| 93 | def rate_limit_jitter() -> float: |
| 94 | return random.uniform(1, 5) |
| 95 | |
| 96 | |
| 97 | class HttpTransport: |
| 98 | def __init__( |
| 99 | self, |
| 100 | *, |
| 101 | max_retries: int = 3, |
| 102 | jitter: Callable[[], float] = rate_limit_jitter, |
| 103 | sleep: Callable[[float], None] = time.sleep, |
| 104 | ) -> None: |
| 105 | self.max_retries = max_retries |
| 106 | self.jitter = jitter |
| 107 | self.sleep = sleep |
| 108 | |
| 109 | def request( |
| 110 | self, |
| 111 | url: str, |
| 112 | *, |
| 113 | body: Mapping[str, Any] | None = None, |
| 114 | headers: Mapping[str, str] | None = None, |
| 115 | method: str = "GET", |
| 116 | ) -> Any: |
| 117 | data = None if body is None else json.dumps(body).encode() |
| 118 | request = urllib.request.Request( |
| 119 | url, |
| 120 | data=data, |
| 121 | headers=dict(headers or {}), |
| 122 | method=method, |
| 123 | ) |
| 124 | for attempt in range(self.max_retries + 1): |
| 125 | try: |
| 126 | with urllib.request.urlopen(request, timeout=30) as response: |
| 127 | response_body = response.read().decode() |
| 128 | break |
| 129 | except urllib.error.HTTPError as error: |
| 130 | response_body = error.read().decode() |
| 131 | if error.code == 429 and attempt < self.max_retries: |
| 132 | delay = self._rate_limit_delay(error, response_body) |
| 133 | print( |
| 134 | "API rate limit reached; " |
| 135 | f"retry {attempt + 1}/{self.max_retries} " |
| 136 | f"in {delay:g} seconds.", |
| 137 | file=sys.stderr, |
| 138 | ) |
| 139 | self.sleep(delay) |
| 140 | continue |
| 141 | message = self._error_message(response_body, error.reason) |
| 142 | raise ApiError( |
| 143 | error.code, |
| 144 | f"API returned {error.code}: {message}", |
| 145 | ) from error |
| 146 | except urllib.error.URLError as error: |
| 147 | raise ApiError( |
| 148 | None, |
| 149 | f"API request failed: {error.reason}", |
| 150 | ) from error |
| 151 | except (OSError, http.client.HTTPException) as error: |
| 152 | raise ApiError(None, f"API request failed: {error}") from error |
| 153 | |
| 154 | if not response_body: |
| 155 | return None |
| 156 | try: |
| 157 | return json.loads(response_body) |
| 158 | except json.JSONDecodeError as error: |
| 159 | raise ApiError(None, "API returned a non-JSON response.") from error |
| 160 | |
| 161 | @staticmethod |
| 162 | def _error_message(response_body: str, fallback: str) -> str: |
| 163 | try: |
| 164 | parsed = json.loads(response_body) |
| 165 | except json.JSONDecodeError: |
| 166 | return fallback |
| 167 | return str(parsed.get("message", fallback)) |
| 168 | |
| 169 | def _rate_limit_delay( |
| 170 | self, |
| 171 | error: urllib.error.HTTPError, |
| 172 | response_body: str, |
| 173 | ) -> float: |
| 174 | try: |
| 175 | parsed = json.loads(response_body) |
| 176 | except json.JSONDecodeError: |
| 177 | parsed = {} |
| 178 | |
| 179 | scope = parsed.get("scope") |
| 180 | reset_header = ( |
| 181 | "RateLimit-User-Reset" if scope == "rest_user" else "RateLimit-Reset" |
| 182 | ) |
| 183 | candidates = [ |
| 184 | error.headers.get(reset_header), |
| 185 | parsed.get("reset"), |
| 186 | ] |
| 187 | for candidate in candidates: |
| 188 | try: |
| 189 | delay = float(candidate) |
| 190 | except (TypeError, ValueError): |
| 191 | continue |
| 192 | if delay >= 0: |
| 193 | return delay + self.jitter() |
| 194 | return 60 + self.jitter() |
| 195 | |
| 196 | |
| 197 | class GitHubClient: |
| 198 | def __init__( |
| 199 | self, |
| 200 | token: str, |
| 201 | repository: str, |
| 202 | transport: HttpTransport | None = None, |
| 203 | ) -> None: |
| 204 | if not token: |
| 205 | raise RuntimeError("GH_TOKEN is not set.") |
| 206 | self.owner, self.repo = repository.split("/", maxsplit=1) |
| 207 | self.transport = transport or HttpTransport() |
| 208 | self.headers = { |
| 209 | "Accept": "application/vnd.github+json", |
| 210 | "Authorization": f"Bearer {token}", |
| 211 | "Content-Type": "application/json", |
| 212 | "User-Agent": "vllm-ci-command", |
| 213 | "X-GitHub-Api-Version": "2022-11-28", |
| 214 | } |
| 215 | |
| 216 | def _request( |
| 217 | self, |
| 218 | path: str, |
| 219 | *, |
| 220 | body: Mapping[str, Any] | None = None, |
| 221 | method: str = "GET", |
| 222 | ) -> Any: |
| 223 | return self.transport.request( |
| 224 | f"https://api.github.com{path}", |
| 225 | body=body, |
| 226 | headers=self.headers, |
| 227 | method=method, |
| 228 | ) |
| 229 | |
| 230 | def _repo_path(self, suffix: str) -> str: |
| 231 | owner = urllib.parse.quote(self.owner, safe="") |
| 232 | repo = urllib.parse.quote(self.repo, safe="") |
| 233 | return f"/repos/{owner}/{repo}{suffix}" |
| 234 | |
| 235 | def _paginate(self, path: str) -> list[dict[str, Any]]: |
| 236 | results: list[dict[str, Any]] = [] |
| 237 | separator = "&" if "?" in path else "?" |
| 238 | for page in range(1, 101): |
| 239 | response = self._request(f"{path}{separator}per_page=100&page={page}") |
| 240 | if not isinstance(response, list): |
| 241 | raise ApiError(None, "GitHub API returned an invalid list response.") |
| 242 | results.extend(response) |
| 243 | if len(response) < 100: |
| 244 | return results |
| 245 | raise ApiError(None, "GitHub API pagination exceeded 10,000 results.") |
| 246 | |
| 247 | def get_pr(self, number: int) -> dict[str, Any]: |
| 248 | return self._request(self._repo_path(f"/pulls/{number}")) |
| 249 | |
| 250 | def get_commits_behind_base(self, base_ref: str, head_sha: str) -> int: |
| 251 | base_ref = urllib.parse.quote(f"refs/heads/{base_ref}", safe="") |
| 252 | head_sha = urllib.parse.quote(head_sha, safe="") |
| 253 | response = self._request( |
| 254 | self._repo_path(f"/compare/{base_ref}...{head_sha}?per_page=1") |
| 255 | ) |
| 256 | behind = response.get("behind_by") if isinstance(response, dict) else None |
| 257 | if type(behind) is not int or behind < 0: |
| 258 | raise ApiError(None, "GitHub returned an invalid commits-behind count.") |
| 259 | return behind |
| 260 | |
| 261 | def list_pulls_for_commit(self, commit: str) -> list[dict[str, Any]]: |
| 262 | commit = urllib.parse.quote(commit, safe="") |
| 263 | response = self._request(self._repo_path(f"/commits/{commit}/pulls")) |
| 264 | if not isinstance(response, list): |
| 265 | raise ApiError(None, "GitHub API returned an invalid pull request list.") |
| 266 | return response |
| 267 | |
| 268 | def get_permission(self, actor: str) -> str: |
| 269 | username = urllib.parse.quote(actor, safe="") |
| 270 | try: |
| 271 | response = self._request( |
| 272 | self._repo_path(f"/collaborators/{username}/permission") |
| 273 | ) |
| 274 | except ApiError as error: |
| 275 | if error.status == 404: |
| 276 | return "none" |
| 277 | raise |
| 278 | return str(response["permission"]) |
| 279 | |
| 280 | def get_review_decision(self, number: int) -> str | None: |
| 281 | query = """ |
| 282 | query($owner: String!, $repo: String!, $number: Int!) { |
| 283 | repository(owner: $owner, name: $repo) { |
| 284 | pullRequest(number: $number) { |
| 285 | reviewDecision |
| 286 | } |
| 287 | } |
| 288 | } |
| 289 | """ |
| 290 | response = self._request( |
| 291 | "/graphql", |
| 292 | body={ |
| 293 | "query": query, |
| 294 | "variables": { |
| 295 | "number": number, |
| 296 | "owner": self.owner, |
| 297 | "repo": self.repo, |
| 298 | }, |
| 299 | }, |
| 300 | method="POST", |
| 301 | ) |
| 302 | return response["data"]["repository"]["pullRequest"]["reviewDecision"] |
| 303 | |
| 304 | def list_reviews(self, number: int) -> list[dict[str, Any]]: |
| 305 | return self._paginate(self._repo_path(f"/pulls/{number}/reviews")) |
| 306 | |
| 307 | def list_issue_comments(self, number: int) -> list[dict[str, Any]]: |
| 308 | return self._paginate(self._repo_path(f"/issues/{number}/comments")) |
| 309 | |
| 310 | def list_reactions(self, comment_id: int) -> list[dict[str, Any]]: |
| 311 | return self._paginate( |
| 312 | self._repo_path(f"/issues/comments/{comment_id}/reactions") |
| 313 | ) |
| 314 | |
| 315 | def add_reaction(self, comment_id: int, content: str) -> None: |
| 316 | self._request( |
| 317 | self._repo_path(f"/issues/comments/{comment_id}/reactions"), |
| 318 | body={"content": content}, |
| 319 | method="POST", |
| 320 | ) |
| 321 | |
| 322 | def add_comment(self, issue_number: int, body: str) -> None: |
| 323 | self._request( |
| 324 | self._repo_path(f"/issues/{issue_number}/comments"), |
| 325 | body={"body": body}, |
| 326 | method="POST", |
| 327 | ) |
| 328 | |
| 329 | |
| 330 | class BuildkiteClient: |
| 331 | def __init__( |
| 332 | self, |
| 333 | token: str, |
| 334 | organization: str, |
| 335 | pipeline: str, |
| 336 | transport: HttpTransport | None = None, |
| 337 | ) -> None: |
| 338 | self.token = token |
| 339 | self.transport = transport or HttpTransport() |
| 340 | organization = urllib.parse.quote(organization, safe="") |
| 341 | pipeline = urllib.parse.quote(pipeline, safe="") |
| 342 | self.base_url = ( |
| 343 | "https://api.buildkite.com/v2/organizations/" |
| 344 | f"{organization}/pipelines/{pipeline}/builds" |
| 345 | ) |
| 346 | |
| 347 | def _headers(self) -> dict[str, str]: |
| 348 | if not self.token: |
| 349 | raise RuntimeError("The BUILDKITE_API_TOKEN repository secret is not set.") |
| 350 | return { |
| 351 | "Authorization": f"Bearer {self.token}", |
| 352 | "Content-Type": "application/json", |
| 353 | "User-Agent": "vllm-ci-command", |
| 354 | } |
| 355 | |
| 356 | def _request_url( |
| 357 | self, |
| 358 | url: str, |
| 359 | *, |
| 360 | body: Mapping[str, Any] | None = None, |
| 361 | method: str = "GET", |
| 362 | ) -> Any: |
| 363 | if ( |
| 364 | url != self.base_url |
| 365 | and not url.startswith(f"{self.base_url}?") |
| 366 | and not url.startswith(f"{self.base_url}/") |
| 367 | ): |
| 368 | raise ApiError(None, "Buildkite API returned an invalid pagination URL.") |
| 369 | return self.transport.request( |
| 370 | url, |
| 371 | body=body, |
| 372 | headers=self._headers(), |
| 373 | method=method, |
| 374 | ) |
| 375 | |
| 376 | def _request( |
| 377 | self, |
| 378 | *, |
| 379 | body: Mapping[str, Any] | None = None, |
| 380 | method: str = "GET", |
| 381 | path: str = "", |
| 382 | query: Sequence[tuple[str, str]] = (), |
| 383 | ) -> Any: |
| 384 | url = f"{self.base_url}{path}" |
| 385 | if query: |
| 386 | url = f"{url}?{urllib.parse.urlencode(query)}" |
| 387 | return self._request_url(url, body=body, method=method) |
| 388 | |
| 389 | def list_builds( |
| 390 | self, |
| 391 | commit: str | None, |
| 392 | *, |
| 393 | branch: str | None = None, |
| 394 | metadata: tuple[str, str] | None = None, |
| 395 | states: Sequence[str] = (), |
| 396 | ) -> list[dict[str, Any]]: |
| 397 | query = [ |
| 398 | ("exclude_jobs", "true"), |
| 399 | ("exclude_pipeline", "true"), |
| 400 | ("per_page", "100"), |
| 401 | ] |
| 402 | if commit: |
| 403 | query.append(("commit", commit)) |
| 404 | if branch: |
| 405 | query.append(("branch", branch)) |
| 406 | if metadata: |
| 407 | key, value = metadata |
| 408 | query.append((f"meta_data[{key}]", value)) |
| 409 | query.extend(("state[]", state) for state in states) |
| 410 | response = self._request(query=query) |
| 411 | if not isinstance(response, list): |
| 412 | raise ApiError(None, "Buildkite API returned an invalid build list.") |
| 413 | return response |
| 414 | |
| 415 | def create_build(self, body: Mapping[str, Any]) -> dict[str, Any]: |
| 416 | return self._request(body=body, method="POST") |
| 417 | |
| 418 | def retry_failed_jobs( |
| 419 | self, |
| 420 | build_number: int, |
| 421 | states: str, |
| 422 | ) -> dict[str, Any]: |
| 423 | number = urllib.parse.quote(str(build_number), safe="") |
| 424 | return self._request( |
| 425 | body={"states": states}, |
| 426 | method="PUT", |
| 427 | path=f"/{number}/retry_failed_jobs", |
| 428 | ) |
| 429 | |
| 430 | def cancel_build(self, build_number: int) -> dict[str, Any]: |
| 431 | number = urllib.parse.quote(str(build_number), safe="") |
| 432 | return self._request(method="PUT", path=f"/{number}/cancel") |
| 433 | |
| 434 | def list_failed_jobs(self, build_number: int) -> list[dict[str, Any]]: |
| 435 | number = urllib.parse.quote(str(build_number), safe="") |
| 436 | query = [ |
| 437 | ("state[]", "failed"), |
| 438 | ("state[]", "timed_out"), |
| 439 | ("state[]", "expired"), |
| 440 | ("include_retried_jobs", "false"), |
| 441 | ("per_page", "100"), |
| 442 | ] |
| 443 | url = f"{self.base_url}/{number}/jobs?{urllib.parse.urlencode(query)}" |
| 444 | jobs: list[dict[str, Any]] = [] |
| 445 | while url: |
| 446 | response = self._request_url(url) |
| 447 | if not isinstance(response, Mapping): |
| 448 | raise ApiError(None, "Buildkite API returned an invalid job list.") |
| 449 | items = response.get("items") |
| 450 | links = response.get("links") |
| 451 | if not isinstance(items, list) or not isinstance(links, Mapping): |
| 452 | raise ApiError(None, "Buildkite API returned an invalid job list.") |
| 453 | jobs.extend(items) |
| 454 | next_url = links.get("next") |
| 455 | if next_url is not None and not isinstance(next_url, str): |
| 456 | raise ApiError( |
| 457 | None, "Buildkite API returned an invalid pagination URL." |
| 458 | ) |
| 459 | url = next_url |
| 460 | return jobs |
| 461 | |
| 462 | |
| 463 | def parse_command(body: str) -> str | None: |
| 464 | if body in ALL_CI_COMMANDS: |
| 465 | return body |
| 466 | return None |
| 467 | |
| 468 | |
| 469 | def pipeline_for_command( |
| 470 | command: str, |
| 471 | *, |
| 472 | amd_ci_pipeline: str = "amd-ci", |
| 473 | upstream_ci_pipeline: str = "ci", |
| 474 | ) -> str: |
| 475 | if command in AMD_CI_COMMANDS: |
| 476 | return amd_ci_pipeline |
| 477 | if command in UPSTREAM_CI_COMMANDS: |
| 478 | return upstream_ci_pipeline |
| 479 | raise ValueError(f"Unsupported CI command: {command}") |
| 480 | |
| 481 | |
| 482 | def ci_name_for_command(command: str) -> str: |
| 483 | if command in AMD_CI_COMMANDS: |
| 484 | return "AMD CI" |
| 485 | if command in UPSTREAM_CI_COMMANDS: |
| 486 | return "CI" |
| 487 | raise ValueError(f"Unsupported CI command: {command}") |
| 488 | |
| 489 | |
| 490 | def run_command_for_command(command: str) -> str: |
| 491 | if command in AMD_CI_COMMANDS: |
| 492 | return COMMAND_RUN_AMD_CI |
| 493 | if command in UPSTREAM_CI_COMMANDS: |
| 494 | return COMMAND_RUN_CI |
| 495 | raise ValueError(f"Unsupported CI command: {command}") |
| 496 | |
| 497 | |
| 498 | def retry_command_for_command(command: str) -> str: |
| 499 | if command in AMD_CI_COMMANDS: |
| 500 | return COMMAND_RETRY_AMD_FAILED |
| 501 | if command in UPSTREAM_CI_COMMANDS: |
| 502 | return COMMAND_RETRY_FAILED |
| 503 | raise ValueError(f"Unsupported CI command: {command}") |
| 504 | |
| 505 | |
| 506 | def parse_trusted_users(value: str = "") -> set[str]: |
| 507 | return { |
| 508 | user.casefold() for item in value.split(",") for user in item.split() if user |
| 509 | } |
| 510 | |
| 511 | |
| 512 | def has_ready_label(pr: Mapping[str, Any]) -> bool: |
| 513 | return any(label["name"] in READY_LABELS for label in pr["labels"]) |
| 514 | |
| 515 | |
| 516 | def is_trusted_permission(permission: str) -> bool: |
| 517 | return permission in TRUSTED_PERMISSIONS |
| 518 | |
| 519 | |
| 520 | def authorize( |
| 521 | *, |
| 522 | actor: str, |
| 523 | permission: str, |
| 524 | pr: Mapping[str, Any], |
| 525 | run_command: str = COMMAND_RUN_CI, |
| 526 | trusted_approval: bool = False, |
| 527 | trusted_users: set[str] | None = None, |
| 528 | ) -> tuple[bool, str]: |
| 529 | trusted_users = trusted_users or set() |
| 530 | if is_trusted_permission(permission): |
| 531 | return True, f"repository {permission} permission" |
| 532 | if actor.casefold() in trusted_users: |
| 533 | return True, "configured trusted contributor" |
| 534 | if actor.casefold() != pr["user"]["login"].casefold(): |
| 535 | return ( |
| 536 | False, |
| 537 | "Only reviewers with write access can use CI commands before CI is " |
| 538 | "delegated to the PR author.", |
| 539 | ) |
| 540 | if pr["draft"]: |
| 541 | return False, "PR authors cannot run CI while the PR is a draft." |
| 542 | if has_ready_label(pr): |
| 543 | return True, "ready label" |
| 544 | if trusted_approval: |
| 545 | return True, "approval from a trusted reviewer" |
| 546 | return ( |
| 547 | False, |
| 548 | f"A reviewer with write access must run `{run_command}`, approve the PR, " |
| 549 | "or add the `ready` label first.", |
| 550 | ) |
| 551 | |
| 552 | |
| 553 | def has_trusted_approval( |
| 554 | github: GitHubClient, |
| 555 | number: int, |
| 556 | trusted_users: set[str], |
| 557 | ) -> bool: |
| 558 | if github.get_review_decision(number) != "APPROVED": |
| 559 | return False |
| 560 | |
| 561 | latest_review_states: dict[str, tuple[str, str]] = {} |
| 562 | for review in github.list_reviews(number): |
| 563 | user = review.get("user") or {} |
| 564 | login = user.get("login") |
| 565 | state = review.get("state") |
| 566 | if login and state in {"APPROVED", "CHANGES_REQUESTED", "DISMISSED"}: |
| 567 | latest_review_states[login.casefold()] = (login, state) |
| 568 | |
| 569 | for login, state in latest_review_states.values(): |
| 570 | if state != "APPROVED": |
| 571 | continue |
| 572 | if login.casefold() in trusted_users: |
| 573 | return True |
| 574 | if is_trusted_permission(github.get_permission(login)): |
| 575 | return True |
| 576 | return False |
| 577 | |
| 578 | |
| 579 | def is_build_for_pr(build: Mapping[str, Any], pr_number: int) -> bool: |
| 580 | pull_request = build.get("pull_request") |
| 581 | if isinstance(pull_request, Mapping): |
| 582 | build_pr_number = pull_request.get("id", pull_request.get("number")) |
| 583 | if build_pr_number is not None: |
| 584 | return str(build_pr_number) == str(pr_number) |
| 585 | metadata = build.get("meta_data") or {} |
| 586 | return str(metadata.get("github-pr-number")) == str(pr_number) |
| 587 | |
| 588 | |
| 589 | def is_active_build(build: Mapping[str, Any]) -> bool: |
| 590 | return bool(build.get("blocked")) or build.get("state") in ACTIVE_BUILD_STATES |
| 591 | |
| 592 | |
| 593 | def is_comment_triggered_build(build: Mapping[str, Any]) -> bool: |
| 594 | metadata = build.get("meta_data") or {} |
| 595 | return bool(metadata.get("github-comment-id")) |
| 596 | |
| 597 | |
| 598 | def blocks_new_run(command: str, build: Mapping[str, Any]) -> bool: |
| 599 | if command in AMD_CI_COMMANDS: |
| 600 | return ( |
| 601 | is_comment_triggered_build(build) |
| 602 | and build.get("state") in ACTIVE_BUILD_STATES |
| 603 | and build.get("state") != "blocked" |
| 604 | ) |
| 605 | return is_active_build(build) |
| 606 | |
| 607 | |
| 608 | def select_latest_build( |
| 609 | builds: Sequence[dict[str, Any]], |
| 610 | pr_number: int, |
| 611 | ) -> dict[str, Any] | None: |
| 612 | matching = [build for build in builds if is_build_for_pr(build, pr_number)] |
| 613 | return max(matching, key=lambda build: build.get("created_at", ""), default=None) |
| 614 | |
| 615 | |
| 616 | def create_build_payload( |
| 617 | *, |
| 618 | actor: str, |
| 619 | comment_id: int, |
| 620 | command: str = COMMAND_RUN_CI, |
| 621 | pr: Mapping[str, Any], |
| 622 | ) -> dict[str, Any]: |
| 623 | if command not in RUN_CI_COMMAND_ENV: |
| 624 | raise ValueError(f"Unsupported run command: {command}") |
| 625 | |
| 626 | env = { |
| 627 | "VLLM_CI_GITHUB_COMMENT_ID": str(comment_id), |
| 628 | "VLLM_CI_TRIGGERED_BY": actor, |
| 629 | **RUN_CI_COMMAND_ENV[command], |
| 630 | } |
| 631 | return { |
| 632 | "commit": pr["head"]["sha"], |
| 633 | # Use the head label (owner:branch) rather than the bare ref so a PR |
| 634 | # whose fork branch is named e.g. "main" does not collide with the |
| 635 | # upstream branch of the same name. |
| 636 | "branch": pr["head"].get("label") or pr["head"]["ref"], |
| 637 | "message": f"PR #{pr['number']} {command} by @{actor}", |
| 638 | "pull_request_id": pr["number"], |
| 639 | "pull_request_base_branch": pr["base"]["ref"], |
| 640 | "pull_request_repository": pr["head"]["repo"]["clone_url"], |
| 641 | "pull_request_labels": [label["name"] for label in pr["labels"]], |
| 642 | "ignore_pipeline_branch_filters": True, |
| 643 | "env": env, |
| 644 | "meta_data": { |
| 645 | "github-comment-id": str(comment_id), |
| 646 | "github-pr-number": str(pr["number"]), |
| 647 | "github-triggered-by": actor, |
| 648 | }, |
| 649 | } |
| 650 | |
| 651 | |
| 652 | def create_retry_build_payload( |
| 653 | *, |
| 654 | actor: str, |
| 655 | comment_id: int, |
| 656 | command: str = COMMAND_RETRY_FAILED, |
| 657 | pr: Mapping[str, Any], |
| 658 | source_build: Mapping[str, Any], |
| 659 | step_keys: Sequence[str], |
| 660 | ) -> dict[str, Any]: |
| 661 | if command not in RETRY_COMMANDS: |
| 662 | raise ValueError(f"Unsupported retry command: {command}") |
| 663 | payload = create_build_payload( |
| 664 | actor=actor, |
| 665 | comment_id=comment_id, |
| 666 | command=run_command_for_command(command), |
| 667 | pr=pr, |
| 668 | ) |
| 669 | source_number = str(source_build["number"]) |
| 670 | payload["message"] = f"PR #{pr['number']} {command} by @{actor}" |
| 671 | payload["env"]["VLLM_CI_ONLY_STEP_KEYS"] = json.dumps( |
| 672 | step_keys, separators=(",", ":") |
| 673 | ) |
| 674 | payload["meta_data"].update( |
| 675 | { |
| 676 | "github-retry-source-build": source_number, |
| 677 | "github-retry-source-commit": str(source_build.get("commit", "")), |
| 678 | } |
| 679 | ) |
| 680 | return payload |
| 681 | |
| 682 | |
| 683 | def add_reaction_safely( |
| 684 | github: GitHubClient, |
| 685 | comment_id: int, |
| 686 | content: str, |
| 687 | ) -> None: |
| 688 | try: |
| 689 | github.add_reaction(comment_id, content) |
| 690 | except Exception as error: |
| 691 | print(f"Could not add {content} reaction: {error}", file=sys.stderr) |
| 692 | |
| 693 | |
| 694 | def command_comment_marker(comment_id: int) -> str: |
| 695 | return f"<!-- vllm-ci-command:{comment_id} -->" |
| 696 | |
| 697 | |
| 698 | def has_bot_comment_marker( |
| 699 | github: GitHubClient, |
| 700 | issue_number: int, |
| 701 | marker: str, |
| 702 | ) -> bool: |
| 703 | return any( |
| 704 | marker in str(comment.get("body", "")) |
| 705 | and (comment.get("user") or {}).get("login") == "github-actions[bot]" |
| 706 | for comment in github.list_issue_comments(issue_number) |
| 707 | ) |
| 708 | |
| 709 | |
| 710 | def is_already_handled( |
| 711 | github: GitHubClient, |
| 712 | issue_number: int, |
| 713 | comment_id: int, |
| 714 | ) -> bool: |
| 715 | terminal_reaction = any( |
| 716 | reaction.get("content") in {"rocket", "-1"} |
| 717 | and (reaction.get("user") or {}).get("login") == "github-actions[bot]" |
| 718 | for reaction in github.list_reactions(comment_id) |
| 719 | ) |
| 720 | if terminal_reaction: |
| 721 | return True |
| 722 | return has_bot_comment_marker( |
| 723 | github, |
| 724 | issue_number, |
| 725 | command_comment_marker(comment_id), |
| 726 | ) |
| 727 | |
| 728 | |
| 729 | def notify_authorized( |
| 730 | event: Mapping[str, Any], |
| 731 | github: GitHubClient, |
| 732 | trusted_users_value: str = "", |
| 733 | ) -> None: |
| 734 | pr = event["pull_request"] |
| 735 | if pr["state"] != "open" or pr["draft"]: |
| 736 | return |
| 737 | |
| 738 | trusted_users = parse_trusted_users(trusted_users_value) |
| 739 | author = pr["user"]["login"] |
| 740 | author_permission = github.get_permission(author) |
| 741 | if ( |
| 742 | is_trusted_permission(author_permission) |
| 743 | or author.casefold() in trusted_users |
| 744 | or has_bot_comment_marker( |
| 745 | github, |
| 746 | pr["number"], |
| 747 | CI_AUTHORIZED_COMMENT_MARKER, |
| 748 | ) |
| 749 | ): |
| 750 | return |
| 751 | |
| 752 | if "label" in event: |
| 753 | if ( |
| 754 | event.get("action") != "labeled" |
| 755 | or event["label"]["name"] not in READY_LABELS |
| 756 | ): |
| 757 | return |
| 758 | elif "review" in event: |
| 759 | if ( |
| 760 | event.get("action") != "submitted" |
| 761 | or str(event["review"].get("state", "")).casefold() != "approved" |
| 762 | or has_ready_label(pr) |
| 763 | or not has_trusted_approval(github, pr["number"], trusted_users) |
| 764 | ): |
| 765 | return |
| 766 | else: |
| 767 | return |
| 768 | |
| 769 | github.add_comment( |
| 770 | pr["number"], |
| 771 | ( |
| 772 | f"✅ @{author}, CI is now available for this PR.\n\n" |
| 773 | "- `/ci run` starts upstream CI; `/amd-ci run` starts AMD CI only.\n" |
| 774 | "- Your branch must contain every commit currently on its upstream " |
| 775 | "target branch. Merge or rebase onto the latest target branch, then " |
| 776 | "rerun the command. Append `--allow-stale` to a run command to test " |
| 777 | "an outdated branch at your own risk.\n" |
| 778 | "- `/ci retry` retries failed jobs in the CI build for the current " |
| 779 | "PR head. If the current head has no CI build, it starts a new CI " |
| 780 | "build for the current head containing only jobs that failed in " |
| 781 | "the latest earlier CI build for this PR.\n" |
| 782 | "- `/amd-ci retry` retries failed jobs in AMD CI for the current PR " |
| 783 | "head. Use `/amd-ci run` when the current head has no AMD CI build.\n" |
| 784 | "- `/ci cancel` cancels scheduled or running CI builds for this PR " |
| 785 | "branch; `/amd-ci cancel` does the same for AMD CI only.\n\n" |
| 786 | f"{CI_AUTHORIZED_COMMENT_MARKER}" |
| 787 | ), |
| 788 | ) |
| 789 | |
| 790 | |
| 791 | def resolve_workflow_run_pr( |
| 792 | workflow_run: Mapping[str, Any], |
| 793 | github: GitHubClient, |
| 794 | ) -> dict[str, Any] | None: |
| 795 | head_sha = str(workflow_run.get("head_sha", "")) |
| 796 | if not head_sha: |
| 797 | return None |
| 798 | |
| 799 | seen: set[int] = set() |
| 800 | |
| 801 | def find_matching_pr( |
| 802 | candidates: Sequence[Mapping[str, Any]], |
| 803 | ) -> dict[str, Any] | None: |
| 804 | for candidate in candidates: |
| 805 | candidate_number = candidate.get("number") |
| 806 | if not isinstance(candidate_number, int) or candidate_number in seen: |
| 807 | continue |
| 808 | seen.add(candidate_number) |
| 809 | try: |
| 810 | pr = github.get_pr(candidate_number) |
| 811 | except ApiError as error: |
| 812 | if error.status == 404: |
| 813 | continue |
| 814 | raise |
| 815 | if pr["state"] == "open" and pr["head"]["sha"] == head_sha: |
| 816 | return pr |
| 817 | return None |
| 818 | |
| 819 | associated_pr = find_matching_pr(workflow_run.get("pull_requests") or []) |
| 820 | if associated_pr is not None: |
| 821 | return associated_pr |
| 822 | return find_matching_pr(github.list_pulls_for_commit(head_sha)) |
| 823 | |
| 824 | |
| 825 | def prepare_pr_for_ci( |
| 826 | github: GitHubClient, |
| 827 | pr: Mapping[str, Any], |
| 828 | command: str, |
| 829 | ) -> tuple[dict[str, Any], str]: |
| 830 | base_ref = pr["base"]["ref"] |
| 831 | no_action = "No new CI build was started." |
| 832 | try: |
| 833 | behind = github.get_commits_behind_base(base_ref, pr["head"]["sha"]) |
| 834 | current_pr = github.get_pr(pr["number"]) |
| 835 | except ApiError as error: |
| 836 | raise CiPreparationError( |
| 837 | f"Could not check the PR against upstream `{base_ref}`. " |
| 838 | f"{no_action} Comment `{command}` again. {error}" |
| 839 | ) from error |
| 840 | if ( |
| 841 | current_pr["state"] != "open" |
| 842 | or current_pr["head"]["sha"] != pr["head"]["sha"] |
| 843 | or current_pr["base"]["ref"] != pr["base"]["ref"] |
| 844 | ): |
| 845 | raise CiPreparationError( |
| 846 | f"The PR changed while checking its branch. {no_action} " |
| 847 | f"Comment `{command}` again." |
| 848 | ) |
| 849 | commits = "commit" if behind == 1 else "commits" |
| 850 | lag = f"This PR is {behind} {commits} behind upstream `{base_ref}`." |
| 851 | if command in ALLOW_STALE_COMMANDS: |
| 852 | warning = "" |
| 853 | if behind: |
| 854 | run_command = command.removesuffix(ALLOW_STALE_SUFFIX) |
| 855 | warning = ( |
| 856 | f"⚠️ {lag} Running CI at your own risk because " |
| 857 | "`--allow-stale` was requested; outdated CI configuration may " |
| 858 | "cause failures. Before merging, merge or rebase onto the latest " |
| 859 | f"`{base_ref}`, then rerun `{run_command}` on the latest PR commit." |
| 860 | ) |
| 861 | print(warning) |
| 862 | return current_pr, warning |
| 863 | |
| 864 | if behind: |
| 865 | raise CiPreparationError( |
| 866 | f"{lag} Your branch must contain every commit currently on upstream " |
| 867 | f"`{base_ref}`. {no_action} " |
| 868 | f"Merge or rebase onto the latest `{base_ref}`, then rerun `{command}`." |
| 869 | " To test this branch at your own risk, " |
| 870 | f"use `{command}{ALLOW_STALE_SUFFIX}`." |
| 871 | ) |
| 872 | return current_pr, "" |
| 873 | |
| 874 | |
| 875 | def handle_run_ci( |
| 876 | *, |
| 877 | actor: str, |
| 878 | buildkite: BuildkiteClient, |
| 879 | comment_id: int, |
| 880 | command: str, |
| 881 | github: GitHubClient, |
| 882 | pr: Mapping[str, Any], |
| 883 | ) -> str: |
| 884 | ci_name = ci_name_for_command(command) |
| 885 | duplicate_builds = buildkite.list_builds( |
| 886 | pr["head"]["sha"], |
| 887 | metadata=("github-comment-id", str(comment_id)), |
| 888 | ) |
| 889 | duplicate = select_latest_build(duplicate_builds, pr["number"]) |
| 890 | if duplicate: |
| 891 | return ( |
| 892 | f"{ci_name} was already requested by this comment: {duplicate['web_url']}" |
| 893 | ) |
| 894 | |
| 895 | current_builds = buildkite.list_builds(pr["head"]["sha"]) |
| 896 | active_build = next( |
| 897 | ( |
| 898 | build |
| 899 | for build in current_builds |
| 900 | if is_build_for_pr(build, pr["number"]) and blocks_new_run(command, build) |
| 901 | ), |
| 902 | None, |
| 903 | ) |
| 904 | if active_build: |
| 905 | return ( |
| 906 | f"{ci_name} is already running for this commit: {active_build['web_url']}" |
| 907 | ) |
| 908 | |
| 909 | current_pr, warning = prepare_pr_for_ci(github, pr, command) |
| 910 | |
| 911 | build = buildkite.create_build( |
| 912 | create_build_payload( |
| 913 | actor=actor, |
| 914 | comment_id=comment_id, |
| 915 | command=command, |
| 916 | pr=current_pr, |
| 917 | ) |
| 918 | ) |
| 919 | return ( |
| 920 | f"Triggered [Buildkite {ci_name} #{build['number']}]({build['web_url']}) " |
| 921 | f"for commit `{current_pr['head']['sha'][:12]}`." |
| 922 | + (f"\n\n{warning}" if warning else "") |
| 923 | ) |
| 924 | |
| 925 | |
| 926 | def handle_retry_failed( |
| 927 | *, |
| 928 | actor: str, |
| 929 | buildkite: BuildkiteClient, |
| 930 | comment_id: int, |
| 931 | command: str, |
| 932 | github: GitHubClient, |
| 933 | pr: Mapping[str, Any], |
| 934 | ) -> str: |
| 935 | ci_name = ci_name_for_command(command) |
| 936 | run_command = run_command_for_command(command) |
| 937 | retry_command = retry_command_for_command(command) |
| 938 | builds = buildkite.list_builds(pr["head"]["sha"]) |
| 939 | build = select_latest_build(builds, pr["number"]) |
| 940 | if build: |
| 941 | metadata = build.get("meta_data") or {} |
| 942 | if str(metadata.get("github-comment-id")) == str(comment_id): |
| 943 | return ( |
| 944 | f"{ci_name} was already requested by this comment: {build['web_url']}" |
| 945 | ) |
| 946 | |
| 947 | retried = buildkite.retry_failed_jobs(build["number"], RETRY_STATES) |
| 948 | if retried["retried_jobs_count"] == 0: |
| 949 | return ( |
| 950 | "No failed, timed-out, or expired jobs need retrying: " |
| 951 | f"{build['web_url']}" |
| 952 | ) |
| 953 | return ( |
| 954 | f"Queued {retried['retried_jobs_count']} failed job(s) for retry in " |
| 955 | f"[Buildkite {ci_name} #{build['number']}]({build['web_url']})." |
| 956 | ) |
| 957 | |
| 958 | if command == COMMAND_RETRY_AMD_FAILED: |
| 959 | return ( |
| 960 | "No AMD CI build exists for the current PR head. " |
| 961 | f"Use `{run_command}` first." |
| 962 | ) |
| 963 | |
| 964 | previous_builds = buildkite.list_builds( |
| 965 | None, |
| 966 | metadata=("github-pr-number", str(pr["number"])), |
| 967 | ) |
| 968 | previous_builds = [ |
| 969 | candidate |
| 970 | for candidate in previous_builds |
| 971 | if candidate.get("commit") != pr["head"]["sha"] |
| 972 | ] |
| 973 | source_build = select_latest_build(previous_builds, pr["number"]) |
| 974 | if not source_build: |
| 975 | return ( |
| 976 | f"No earlier {ci_name} build exists for this PR. Use `{run_command}` first." |
| 977 | ) |
| 978 | if not source_build.get("finished_at") or is_active_build(source_build): |
| 979 | return ( |
| 980 | f"The previous {ci_name} build is still running: {source_build['web_url']}" |
| 981 | ) |
| 982 | |
| 983 | failed_jobs = buildkite.list_failed_jobs(source_build["number"]) |
| 984 | failed_script_jobs = [job for job in failed_jobs if job.get("type") == "script"] |
| 985 | missing_step_keys = [job for job in failed_script_jobs if not job.get("step_key")] |
| 986 | if missing_step_keys: |
| 987 | return ( |
| 988 | f"[Buildkite {ci_name} #{source_build['number']}]" |
| 989 | f"({source_build['web_url']}) has failed jobs without stable step " |
| 990 | "keys, so they cannot be retried on a new commit. " |
| 991 | f"Use `{run_command}`." |
| 992 | ) |
| 993 | |
| 994 | failed_step_keys = {str(job["step_key"]) for job in failed_script_jobs} |
| 995 | setup_failures = sorted( |
| 996 | step_key |
| 997 | for step_key in failed_step_keys |
| 998 | if step_key.startswith("image-build") or step_key in SETUP_STEP_KEYS |
| 999 | ) |
| 1000 | if setup_failures: |
| 1001 | return ( |
| 1002 | f"[Buildkite {ci_name} #{source_build['number']}]" |
| 1003 | f"({source_build['web_url']}) failed during CI setup, so its test " |
| 1004 | f"failure set is incomplete. Use `{run_command}` for the new commit." |
| 1005 | ) |
| 1006 | |
| 1007 | step_keys = sorted(failed_step_keys) |
| 1008 | if not step_keys: |
| 1009 | return ( |
| 1010 | "No failed, timed-out, or expired jobs need retrying in " |
| 1011 | f"[Buildkite {ci_name} #{source_build['number']}]" |
| 1012 | f"({source_build['web_url']})." |
| 1013 | ) |
| 1014 | |
| 1015 | current_pr = github.get_pr(pr["number"]) |
| 1016 | if current_pr["state"] != "open" or current_pr["head"]["sha"] != pr["head"]["sha"]: |
| 1017 | return ( |
| 1018 | "The PR head changed while processing the command. " |
| 1019 | f"Comment `{retry_command}` again." |
| 1020 | ) |
| 1021 | |
| 1022 | retry_build = buildkite.create_build( |
| 1023 | create_retry_build_payload( |
| 1024 | actor=actor, |
| 1025 | comment_id=comment_id, |
| 1026 | command=command, |
| 1027 | pr=current_pr, |
| 1028 | source_build=source_build, |
| 1029 | step_keys=step_keys, |
| 1030 | ) |
| 1031 | ) |
| 1032 | return ( |
| 1033 | f"Triggered [Buildkite {ci_name} #{retry_build['number']}]" |
| 1034 | f"({retry_build['web_url']}) for commit " |
| 1035 | f"`{current_pr['head']['sha'][:12]}`, running {len(step_keys)} failed " |
| 1036 | f"step(s) from [Buildkite {ci_name} #{source_build['number']}]" |
| 1037 | f"({source_build['web_url']})." |
| 1038 | ) |
| 1039 | |
| 1040 | |
| 1041 | def handle_cancel_ci( |
| 1042 | *, |
| 1043 | buildkite: BuildkiteClient, |
| 1044 | command: str = COMMAND_CANCEL_CI, |
| 1045 | pr: Mapping[str, Any], |
| 1046 | ) -> str: |
| 1047 | ci_name = ci_name_for_command(command) |
| 1048 | branch = pr["head"]["ref"] |
| 1049 | # Builds are created under the head label (owner:branch); also match the |
| 1050 | # bare ref so builds created before that change can still be cancelled. |
| 1051 | branches = [branch] |
| 1052 | head_label = pr["head"].get("label") |
| 1053 | if head_label and head_label not in branches: |
| 1054 | branches.insert(0, str(head_label)) |
| 1055 | states = CANCELABLE_BUILD_STATES |
| 1056 | |
| 1057 | builds_by_number: dict[int, dict[str, Any]] = {} |
| 1058 | for candidate_branch in branches: |
| 1059 | for build in buildkite.list_builds( |
| 1060 | None, |
| 1061 | branch=candidate_branch, |
| 1062 | states=states, |
| 1063 | ): |
| 1064 | builds_by_number[build["number"]] = build |
| 1065 | |
| 1066 | cancelable_builds = [ |
| 1067 | build |
| 1068 | for build in builds_by_number.values() |
| 1069 | if build.get("branch") in branches |
| 1070 | and is_build_for_pr(build, pr["number"]) |
| 1071 | and build.get("state") in states |
| 1072 | ] |
| 1073 | if not cancelable_builds: |
| 1074 | return f"No cancelable {ci_name} build is running for branch `{branch}`." |
| 1075 | |
| 1076 | for build in cancelable_builds: |
| 1077 | buildkite.cancel_build(build["number"]) |
| 1078 | |
| 1079 | links = ", ".join( |
| 1080 | f"[#{build['number']}]({build['web_url']})" for build in cancelable_builds |
| 1081 | ) |
| 1082 | count = len(cancelable_builds) |
| 1083 | noun = "build" if count == 1 else "builds" |
| 1084 | return ( |
| 1085 | f"Requested cancellation of {count} {ci_name} {noun} for `{branch}`: {links}." |
| 1086 | ) |
| 1087 | |
| 1088 | |
| 1089 | def run( |
| 1090 | event: Mapping[str, Any], |
| 1091 | github: GitHubClient, |
| 1092 | buildkite: BuildkiteClient, |
| 1093 | trusted_users_value: str = "", |
| 1094 | ) -> None: |
| 1095 | command = parse_command(event["comment"]["body"]) |
| 1096 | if not command or "pull_request" not in event["issue"]: |
| 1097 | return |
| 1098 | |
| 1099 | issue_number = event["issue"]["number"] |
| 1100 | comment_id = event["comment"]["id"] |
| 1101 | actor = event["comment"]["user"]["login"] |
| 1102 | |
| 1103 | if is_already_handled(github, issue_number, comment_id): |
| 1104 | print(f"Comment {comment_id} was already handled.") |
| 1105 | return |
| 1106 | add_reaction_safely(github, comment_id, "eyes") |
| 1107 | |
| 1108 | try: |
| 1109 | pr = github.get_pr(issue_number) |
| 1110 | permission = github.get_permission(actor) |
| 1111 | if pr["state"] != "open": |
| 1112 | github.add_comment( |
| 1113 | issue_number, |
| 1114 | "❌ CI commands require an open PR.\n\n" |
| 1115 | f"{command_comment_marker(comment_id)}", |
| 1116 | ) |
| 1117 | return |
| 1118 | |
| 1119 | trusted_users = parse_trusted_users(trusted_users_value) |
| 1120 | should_check_approval = ( |
| 1121 | not is_trusted_permission(permission) |
| 1122 | and actor.casefold() not in trusted_users |
| 1123 | and actor.casefold() == pr["user"]["login"].casefold() |
| 1124 | and not pr["draft"] |
| 1125 | and not has_ready_label(pr) |
| 1126 | ) |
| 1127 | trusted_approval = should_check_approval and has_trusted_approval( |
| 1128 | github, |
| 1129 | issue_number, |
| 1130 | trusted_users, |
| 1131 | ) |
| 1132 | allowed, reason = authorize( |
| 1133 | actor=actor, |
| 1134 | permission=permission, |
| 1135 | pr=pr, |
| 1136 | run_command=run_command_for_command(command), |
| 1137 | trusted_approval=trusted_approval, |
| 1138 | trusted_users=trusted_users, |
| 1139 | ) |
| 1140 | if not allowed: |
| 1141 | github.add_comment( |
| 1142 | issue_number, |
| 1143 | f"❌ @{actor}, {reason}\n\n{command_comment_marker(comment_id)}", |
| 1144 | ) |
| 1145 | return |
| 1146 | |
| 1147 | print(f"Authorized @{actor}: {reason}") |
| 1148 | if command in RUN_CI_COMMAND_ENV: |
| 1149 | message = handle_run_ci( |
| 1150 | actor=actor, |
| 1151 | buildkite=buildkite, |
| 1152 | comment_id=comment_id, |
| 1153 | command=command, |
| 1154 | github=github, |
| 1155 | pr=pr, |
| 1156 | ) |
| 1157 | elif command in RETRY_COMMANDS: |
| 1158 | message = handle_retry_failed( |
| 1159 | actor=actor, |
| 1160 | buildkite=buildkite, |
| 1161 | comment_id=comment_id, |
| 1162 | command=command, |
| 1163 | github=github, |
| 1164 | pr=pr, |
| 1165 | ) |
| 1166 | elif command in CANCEL_COMMANDS: |
| 1167 | message = handle_cancel_ci( |
| 1168 | buildkite=buildkite, |
| 1169 | command=command, |
| 1170 | pr=pr, |
| 1171 | ) |
| 1172 | else: |
| 1173 | raise ValueError(f"Unsupported CI command: {command}") |
| 1174 | add_reaction_safely(github, comment_id, "rocket") |
| 1175 | github.add_comment(issue_number, f"✅ {message}") |
| 1176 | except CiPreparationError as error: |
| 1177 | add_reaction_safely(github, comment_id, "-1") |
| 1178 | github.add_comment( |
| 1179 | issue_number, |
| 1180 | f"❌ {error}\n\n{command_comment_marker(comment_id)}", |
| 1181 | ) |
| 1182 | except Exception: |
| 1183 | add_reaction_safely(github, comment_id, "confused") |
| 1184 | raise |
| 1185 | |
| 1186 | |
| 1187 | def main() -> None: |
| 1188 | event_path = os.environ["GITHUB_EVENT_PATH"] |
| 1189 | with open(event_path, encoding="utf-8") as event_file: |
| 1190 | event = json.load(event_file) |
| 1191 | |
| 1192 | github = GitHubClient( |
| 1193 | os.environ.get("GH_TOKEN", ""), |
| 1194 | os.environ["GITHUB_REPOSITORY"], |
| 1195 | ) |
| 1196 | event_name = os.environ.get("GITHUB_EVENT_NAME", "issue_comment") |
| 1197 | if event_name == "pull_request_target": |
| 1198 | notify_authorized( |
| 1199 | event, |
| 1200 | github, |
| 1201 | os.environ.get("CI_TRUSTED_USERS", ""), |
| 1202 | ) |
| 1203 | return |
| 1204 | if event_name == "workflow_run": |
| 1205 | pr = resolve_workflow_run_pr(event["workflow_run"], github) |
| 1206 | if pr is None: |
| 1207 | print("Could not resolve an open PR for the approval workflow run.") |
| 1208 | return |
| 1209 | notify_authorized( |
| 1210 | { |
| 1211 | "action": "submitted", |
| 1212 | "pull_request": pr, |
| 1213 | "review": {"state": "approved"}, |
| 1214 | }, |
| 1215 | github, |
| 1216 | os.environ.get("CI_TRUSTED_USERS", ""), |
| 1217 | ) |
| 1218 | return |
| 1219 | |
| 1220 | command = parse_command(event["comment"]["body"]) |
| 1221 | if not command: |
| 1222 | return |
| 1223 | |
| 1224 | pipeline = pipeline_for_command( |
| 1225 | command, |
| 1226 | amd_ci_pipeline=os.environ.get("BUILDKITE_AMD_PIPELINE", "amd-ci"), |
| 1227 | upstream_ci_pipeline=os.environ.get("BUILDKITE_PIPELINE", "ci"), |
| 1228 | ) |
| 1229 | buildkite = BuildkiteClient( |
| 1230 | os.environ.get("BUILDKITE_API_TOKEN", ""), |
| 1231 | os.environ.get("BUILDKITE_ORGANIZATION", "vllm"), |
| 1232 | pipeline, |
| 1233 | ) |
| 1234 | run( |
| 1235 | event, |
| 1236 | github, |
| 1237 | buildkite, |
| 1238 | os.environ.get("CI_TRUSTED_USERS", ""), |
| 1239 | ) |
| 1240 | |
| 1241 | |
| 1242 | if __name__ == "__main__": |
| 1243 | main() |