| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import io |
| 5 | import json |
| 6 | import unittest |
| 7 | import urllib.error |
| 8 | from typing import Any |
| 9 | from unittest.mock import patch |
| 10 | |
| 11 | from run_ci_command import ( |
| 12 | CANCELABLE_BUILD_STATES, |
| 13 | CI_AUTHORIZED_COMMENT_MARKER, |
| 14 | COMMAND_CANCEL_AMD_CI, |
| 15 | COMMAND_CANCEL_CI, |
| 16 | COMMAND_RETRY_AMD_FAILED, |
| 17 | COMMAND_RETRY_FAILED, |
| 18 | COMMAND_RUN_AMD_CI, |
| 19 | COMMAND_RUN_AMD_CI_ALL, |
| 20 | COMMAND_RUN_AMD_CI_NIGHTLY, |
| 21 | COMMAND_RUN_CI, |
| 22 | COMMAND_RUN_CI_ALL, |
| 23 | COMMAND_RUN_CI_NIGHTLY, |
| 24 | RETRY_STATES, |
| 25 | RUN_CI_COMMAND_ENV, |
| 26 | ApiError, |
| 27 | BuildkiteClient, |
| 28 | GitHubClient, |
| 29 | HttpTransport, |
| 30 | authorize, |
| 31 | create_build_payload, |
| 32 | has_trusted_approval, |
| 33 | is_active_build, |
| 34 | is_build_for_pr, |
| 35 | notify_authorized, |
| 36 | parse_command, |
| 37 | parse_trusted_users, |
| 38 | pipeline_for_command, |
| 39 | resolve_workflow_run_pr, |
| 40 | run, |
| 41 | select_latest_build, |
| 42 | ) |
| 43 | |
| 44 | |
| 45 | def make_pr(**overrides: Any) -> dict[str, Any]: |
| 46 | pr = { |
| 47 | "base": {"ref": "main"}, |
| 48 | "draft": False, |
| 49 | "head": { |
| 50 | "label": "contributor:feature", |
| 51 | "ref": "feature", |
| 52 | "repo": {"clone_url": "https://github.com/contributor/vllm.git"}, |
| 53 | "sha": "0123456789abcdef", |
| 54 | }, |
| 55 | "labels": [], |
| 56 | "number": 42, |
| 57 | "state": "open", |
| 58 | "user": {"login": "author"}, |
| 59 | } |
| 60 | pr.update(overrides) |
| 61 | return pr |
| 62 | |
| 63 | |
| 64 | def make_event(command: str, actor: str = "reviewer") -> dict[str, Any]: |
| 65 | return { |
| 66 | "comment": { |
| 67 | "body": command, |
| 68 | "id": 99, |
| 69 | "user": {"login": actor}, |
| 70 | }, |
| 71 | "issue": { |
| 72 | "number": 42, |
| 73 | "pull_request": {}, |
| 74 | }, |
| 75 | } |
| 76 | |
| 77 | |
| 78 | class FakeGitHub: |
| 79 | def __init__( |
| 80 | self, |
| 81 | *, |
| 82 | permission: str = "write", |
| 83 | permissions: dict[str, str] | None = None, |
| 84 | pr: dict[str, Any] | None = None, |
| 85 | review_decision: str = "REVIEW_REQUIRED", |
| 86 | reviews: list[dict[str, Any]] | None = None, |
| 87 | comments: list[str] | None = None, |
| 88 | pulls_for_commit: list[dict[str, Any]] | None = None, |
| 89 | prs: dict[int, dict[str, Any]] | None = None, |
| 90 | behind: int = 0, |
| 91 | ) -> None: |
| 92 | self.comments = comments or [] |
| 93 | self.permission = permission |
| 94 | self.permissions = permissions or {} |
| 95 | self.pr = pr or make_pr() |
| 96 | self.reactions: list[str] = [] |
| 97 | self.review_decision = review_decision |
| 98 | self.reviews = reviews or [] |
| 99 | self.pulls_for_commit = pulls_for_commit or [] |
| 100 | self.prs = prs or {self.pr["number"]: self.pr} |
| 101 | self.behind = behind |
| 102 | self.lag_queries: list[tuple[str, str]] = [] |
| 103 | |
| 104 | def get_commits_behind_base(self, base_ref: str, head_sha: str) -> int: |
| 105 | self.lag_queries.append((base_ref, head_sha)) |
| 106 | return self.behind |
| 107 | |
| 108 | def get_pr(self, number: int) -> dict[str, Any]: |
| 109 | try: |
| 110 | return self.prs[number] |
| 111 | except KeyError as error: |
| 112 | raise ApiError(404, "Not found") from error |
| 113 | |
| 114 | def get_permission(self, actor: str) -> str: |
| 115 | return self.permissions.get(actor, self.permission) |
| 116 | |
| 117 | def list_pulls_for_commit(self, commit: str) -> list[dict[str, Any]]: |
| 118 | return self.pulls_for_commit |
| 119 | |
| 120 | def get_review_decision(self, number: int) -> str: |
| 121 | return self.review_decision |
| 122 | |
| 123 | def list_reviews(self, number: int) -> list[dict[str, Any]]: |
| 124 | return self.reviews |
| 125 | |
| 126 | def list_issue_comments(self, number: int) -> list[dict[str, Any]]: |
| 127 | return [ |
| 128 | { |
| 129 | "body": body, |
| 130 | "user": {"login": "github-actions[bot]"}, |
| 131 | } |
| 132 | for body in self.comments |
| 133 | ] |
| 134 | |
| 135 | def list_reactions(self, comment_id: int) -> list[dict[str, Any]]: |
| 136 | return [] |
| 137 | |
| 138 | def add_reaction(self, comment_id: int, content: str) -> None: |
| 139 | self.reactions.append(content) |
| 140 | |
| 141 | def add_comment(self, issue_number: int, body: str) -> None: |
| 142 | self.comments.append(body) |
| 143 | |
| 144 | |
| 145 | class FakeBuildkite: |
| 146 | def __init__( |
| 147 | self, |
| 148 | build_lists: list[list[dict[str, Any]]] | None = None, |
| 149 | failed_job_lists: list[list[dict[str, Any]]] | None = None, |
| 150 | ) -> None: |
| 151 | self.build_lists = build_lists or [] |
| 152 | self.created_builds: list[dict[str, Any]] = [] |
| 153 | self.failed_job_lists = failed_job_lists or [] |
| 154 | self.job_list_calls: list[int] = [] |
| 155 | self.list_calls: list[tuple[str | None, tuple[str, str] | None]] = [] |
| 156 | self.list_requests: list[dict[str, Any]] = [] |
| 157 | self.retry_calls: list[tuple[int, str]] = [] |
| 158 | self.cancel_calls: list[int] = [] |
| 159 | |
| 160 | def list_builds( |
| 161 | self, |
| 162 | commit: str | None, |
| 163 | *, |
| 164 | branch: str | None = None, |
| 165 | metadata: tuple[str, str] | None = None, |
| 166 | states: tuple[str, ...] = (), |
| 167 | ) -> list[dict[str, Any]]: |
| 168 | self.list_calls.append((commit, metadata)) |
| 169 | self.list_requests.append( |
| 170 | { |
| 171 | "branch": branch, |
| 172 | "commit": commit, |
| 173 | "metadata": metadata, |
| 174 | "states": states, |
| 175 | } |
| 176 | ) |
| 177 | return self.build_lists.pop(0) |
| 178 | |
| 179 | def create_build(self, body: dict[str, Any]) -> dict[str, Any]: |
| 180 | self.created_builds.append(body) |
| 181 | return { |
| 182 | "number": 123, |
| 183 | "web_url": "https://buildkite.example/builds/123", |
| 184 | } |
| 185 | |
| 186 | def retry_failed_jobs( |
| 187 | self, |
| 188 | build_number: int, |
| 189 | states: str, |
| 190 | ) -> dict[str, Any]: |
| 191 | self.retry_calls.append((build_number, states)) |
| 192 | return {"retried_jobs_count": 3} |
| 193 | |
| 194 | def cancel_build(self, build_number: int) -> dict[str, Any]: |
| 195 | self.cancel_calls.append(build_number) |
| 196 | return {"number": build_number, "state": "canceling"} |
| 197 | |
| 198 | def list_failed_jobs(self, build_number: int) -> list[dict[str, Any]]: |
| 199 | self.job_list_calls.append(build_number) |
| 200 | return self.failed_job_lists.pop(0) |
| 201 | |
| 202 | |
| 203 | class FakeTransport: |
| 204 | def __init__(self, response: Any) -> None: |
| 205 | self.calls: list[dict[str, Any]] = [] |
| 206 | self.response = response |
| 207 | |
| 208 | def request(self, url: str, **kwargs: Any) -> Any: |
| 209 | self.calls.append({"url": url, **kwargs}) |
| 210 | if isinstance(self.response, tuple): |
| 211 | return self.response[len(self.calls) - 1] |
| 212 | return self.response |
| 213 | |
| 214 | |
| 215 | class FakeHttpResponse: |
| 216 | def __init__(self, response: Any) -> None: |
| 217 | self.response = response |
| 218 | |
| 219 | def __enter__(self) -> "FakeHttpResponse": |
| 220 | return self |
| 221 | |
| 222 | def __exit__(self, *args: Any) -> None: |
| 223 | return None |
| 224 | |
| 225 | def read(self) -> bytes: |
| 226 | return json.dumps(self.response).encode() |
| 227 | |
| 228 | |
| 229 | class RunCiCommandTest(unittest.TestCase): |
| 230 | @patch("run_ci_command.urllib.request.urlopen") |
| 231 | def test_http_transport_retries_buildkite_rate_limit(self, urlopen: Any) -> None: |
| 232 | body = { |
| 233 | "message": "Please wait 9 seconds before making more requests.", |
| 234 | "reset": 9, |
| 235 | "scope": "rest", |
| 236 | } |
| 237 | rate_limit_error = urllib.error.HTTPError( |
| 238 | "https://api.buildkite.com/v2/builds", |
| 239 | 429, |
| 240 | "Too Many Requests", |
| 241 | { |
| 242 | "RateLimit-Limit": "400", |
| 243 | "RateLimit-Remaining": "0", |
| 244 | "RateLimit-Reset": "9", |
| 245 | }, |
| 246 | io.BytesIO(json.dumps(body).encode()), |
| 247 | ) |
| 248 | urlopen.side_effect = [rate_limit_error, FakeHttpResponse({"ok": True})] |
| 249 | delays: list[float] = [] |
| 250 | |
| 251 | response = HttpTransport( |
| 252 | jitter=lambda: 2.5, |
| 253 | sleep=delays.append, |
| 254 | ).request("https://api.buildkite.com/v2/builds") |
| 255 | |
| 256 | self.assertEqual(response, {"ok": True}) |
| 257 | self.assertEqual(delays, [11.5]) |
| 258 | self.assertEqual(urlopen.call_count, 2) |
| 259 | |
| 260 | @patch("run_ci_command.urllib.request.urlopen") |
| 261 | def test_http_transport_retries_rate_limit_three_times(self, urlopen: Any) -> None: |
| 262 | def rate_limit_error() -> urllib.error.HTTPError: |
| 263 | body = { |
| 264 | "message": "Please wait 9 seconds before making more requests.", |
| 265 | "reset": 9, |
| 266 | "scope": "rest", |
| 267 | } |
| 268 | return urllib.error.HTTPError( |
| 269 | "https://api.buildkite.com/v2/builds", |
| 270 | 429, |
| 271 | "Too Many Requests", |
| 272 | { |
| 273 | "RateLimit-Limit": "400", |
| 274 | "RateLimit-Remaining": "0", |
| 275 | "RateLimit-Reset": "9", |
| 276 | }, |
| 277 | io.BytesIO(json.dumps(body).encode()), |
| 278 | ) |
| 279 | |
| 280 | urlopen.side_effect = [rate_limit_error() for _ in range(4)] |
| 281 | delays: list[float] = [] |
| 282 | |
| 283 | with self.assertRaisesRegex(RuntimeError, "API returned 429"): |
| 284 | HttpTransport( |
| 285 | jitter=lambda: 2, |
| 286 | sleep=delays.append, |
| 287 | ).request("https://api.buildkite.com/v2/builds") |
| 288 | |
| 289 | self.assertEqual(delays, [11, 11, 11]) |
| 290 | self.assertEqual(urlopen.call_count, 4) |
| 291 | |
| 292 | @patch("run_ci_command.urllib.request.urlopen") |
| 293 | def test_http_transport_does_not_retry_permission_error(self, urlopen: Any) -> None: |
| 294 | permission_error = urllib.error.HTTPError( |
| 295 | "https://api.github.com/repos/vllm-project/vllm/issues/1/comments", |
| 296 | 403, |
| 297 | "Forbidden", |
| 298 | {}, |
| 299 | io.BytesIO(b'{"message":"Resource not accessible by integration"}'), |
| 300 | ) |
| 301 | urlopen.side_effect = permission_error |
| 302 | delays: list[float] = [] |
| 303 | |
| 304 | with self.assertRaisesRegex( |
| 305 | RuntimeError, |
| 306 | "Resource not accessible by integration", |
| 307 | ): |
| 308 | HttpTransport( |
| 309 | jitter=lambda: 2, |
| 310 | sleep=delays.append, |
| 311 | ).request( |
| 312 | "https://api.github.com/repos/vllm-project/vllm/issues/1/comments" |
| 313 | ) |
| 314 | |
| 315 | self.assertEqual(delays, []) |
| 316 | self.assertEqual(urlopen.call_count, 1) |
| 317 | |
| 318 | def test_only_exact_ci_commands_are_accepted(self) -> None: |
| 319 | commands = ( |
| 320 | COMMAND_RUN_CI, |
| 321 | COMMAND_RUN_CI_ALL, |
| 322 | COMMAND_RUN_CI_NIGHTLY, |
| 323 | COMMAND_RETRY_FAILED, |
| 324 | COMMAND_CANCEL_CI, |
| 325 | COMMAND_RUN_AMD_CI, |
| 326 | COMMAND_RUN_AMD_CI_ALL, |
| 327 | COMMAND_RUN_AMD_CI_NIGHTLY, |
| 328 | COMMAND_RETRY_AMD_FAILED, |
| 329 | COMMAND_CANCEL_AMD_CI, |
| 330 | ) |
| 331 | for command in commands: |
| 332 | with self.subTest(command=command): |
| 333 | self.assertEqual(parse_command(command), command) |
| 334 | override = f"{command} --allow-stale" |
| 335 | if " run" in command: |
| 336 | self.assertEqual(parse_command(override), override) |
| 337 | else: |
| 338 | self.assertIsNone(parse_command(override)) |
| 339 | |
| 340 | self.assertIsNone(parse_command("/ci run please")) |
| 341 | self.assertIsNone(parse_command("/ci run all please")) |
| 342 | self.assertIsNone(parse_command("/ci cancel please")) |
| 343 | self.assertIsNone(parse_command(" /ci run")) |
| 344 | self.assertIsNone(parse_command("/amd-ci run please")) |
| 345 | self.assertIsNone(parse_command("/amd-ci retry ")) |
| 346 | self.assertIsNone(parse_command("/AMD-CI run")) |
| 347 | self.assertIsNone(parse_command("/amdci run")) |
| 348 | self.assertIsNone(parse_command("/ci run --allow-stale all")) |
| 349 | self.assertIsNone(parse_command("/amd-ci run --allow-stale please")) |
| 350 | |
| 351 | def test_commands_select_only_their_configured_pipeline(self) -> None: |
| 352 | cases = ( |
| 353 | (COMMAND_RUN_CI, "ci"), |
| 354 | (COMMAND_RUN_CI_ALL, "ci"), |
| 355 | (COMMAND_RUN_CI_NIGHTLY, "ci"), |
| 356 | (COMMAND_RETRY_FAILED, "ci"), |
| 357 | (COMMAND_CANCEL_CI, "ci"), |
| 358 | (COMMAND_RUN_AMD_CI, "amd-ci"), |
| 359 | (COMMAND_RUN_AMD_CI_ALL, "amd-ci"), |
| 360 | (COMMAND_RUN_AMD_CI_NIGHTLY, "amd-ci"), |
| 361 | (COMMAND_RETRY_AMD_FAILED, "amd-ci"), |
| 362 | (COMMAND_CANCEL_AMD_CI, "amd-ci"), |
| 363 | ) |
| 364 | for command, expected_pipeline in cases: |
| 365 | with self.subTest(command=command): |
| 366 | self.assertEqual( |
| 367 | pipeline_for_command(command), |
| 368 | expected_pipeline, |
| 369 | ) |
| 370 | with self.assertRaisesRegex(ValueError, "Unsupported CI command"): |
| 371 | pipeline_for_command("/amd-ci run arbitrary-pipeline") |
| 372 | |
| 373 | def test_write_access_authorizes_reviewers_and_authors(self) -> None: |
| 374 | allowed, _ = authorize( |
| 375 | actor="reviewer", |
| 376 | permission="write", |
| 377 | pr=make_pr(), |
| 378 | ) |
| 379 | self.assertTrue(allowed) |
| 380 | |
| 381 | def test_configured_trusted_contributors_can_run_ci(self) -> None: |
| 382 | trusted_users = parse_trusted_users("trusted-one, TRUSTED-TWO") |
| 383 | allowed, _ = authorize( |
| 384 | actor="trusted-two", |
| 385 | permission="read", |
| 386 | pr=make_pr(), |
| 387 | trusted_users=trusted_users, |
| 388 | ) |
| 389 | self.assertTrue(allowed) |
| 390 | |
| 391 | def test_authors_need_an_approval_or_ready_label(self) -> None: |
| 392 | pending, _ = authorize( |
| 393 | actor="author", |
| 394 | permission="read", |
| 395 | pr=make_pr(), |
| 396 | ) |
| 397 | approved, _ = authorize( |
| 398 | actor="author", |
| 399 | permission="read", |
| 400 | pr=make_pr(), |
| 401 | trusted_approval=True, |
| 402 | ) |
| 403 | ready, _ = authorize( |
| 404 | actor="author", |
| 405 | permission="read", |
| 406 | pr=make_pr(labels=[{"name": "ready"}]), |
| 407 | ) |
| 408 | self.assertFalse(pending) |
| 409 | self.assertTrue(approved) |
| 410 | self.assertTrue(ready) |
| 411 | |
| 412 | def test_non_author_contributors_without_write_are_denied(self) -> None: |
| 413 | allowed, _ = authorize( |
| 414 | actor="contributor", |
| 415 | permission="read", |
| 416 | pr=make_pr(), |
| 417 | trusted_approval=True, |
| 418 | ) |
| 419 | self.assertFalse(allowed) |
| 420 | |
| 421 | def test_authors_cannot_use_ready_state_on_draft_prs(self) -> None: |
| 422 | allowed, _ = authorize( |
| 423 | actor="author", |
| 424 | permission="read", |
| 425 | pr=make_pr(draft=True, labels=[{"name": "ready"}]), |
| 426 | trusted_approval=True, |
| 427 | ) |
| 428 | self.assertFalse(allowed) |
| 429 | |
| 430 | def test_only_trusted_reviewers_can_delegate_through_approval(self) -> None: |
| 431 | approved_review = { |
| 432 | "state": "APPROVED", |
| 433 | "user": {"login": "reviewer"}, |
| 434 | } |
| 435 | trusted = FakeGitHub( |
| 436 | permission="read", |
| 437 | permissions={"reviewer": "write"}, |
| 438 | review_decision="APPROVED", |
| 439 | reviews=[approved_review], |
| 440 | ) |
| 441 | untrusted = FakeGitHub( |
| 442 | permission="read", |
| 443 | review_decision="APPROVED", |
| 444 | reviews=[approved_review], |
| 445 | ) |
| 446 | self.assertTrue(has_trusted_approval(trusted, 42, set())) |
| 447 | self.assertFalse(has_trusted_approval(untrusted, 42, set())) |
| 448 | |
| 449 | def test_build_matching_is_scoped_to_the_pr(self) -> None: |
| 450 | self.assertTrue(is_build_for_pr({"pull_request": {"id": 42}}, 42)) |
| 451 | self.assertFalse(is_build_for_pr({"pull_request": {"id": 43}}, 42)) |
| 452 | self.assertTrue( |
| 453 | is_build_for_pr( |
| 454 | {"meta_data": {"github-pr-number": "42"}}, |
| 455 | 42, |
| 456 | ) |
| 457 | ) |
| 458 | |
| 459 | def test_latest_build_selection_ignores_other_prs(self) -> None: |
| 460 | latest = select_latest_build( |
| 461 | [ |
| 462 | { |
| 463 | "created_at": "2026-07-28T02:00:00Z", |
| 464 | "number": 3, |
| 465 | "pull_request": {"id": 43}, |
| 466 | }, |
| 467 | { |
| 468 | "created_at": "2026-07-28T01:00:00Z", |
| 469 | "number": 2, |
| 470 | "pull_request": {"id": 42}, |
| 471 | }, |
| 472 | { |
| 473 | "created_at": "2026-07-28T00:00:00Z", |
| 474 | "number": 1, |
| 475 | "pull_request": {"id": 42}, |
| 476 | }, |
| 477 | ], |
| 478 | 42, |
| 479 | ) |
| 480 | self.assertEqual(latest["number"], 2) |
| 481 | |
| 482 | def test_active_build_states_prevent_duplicate_runs(self) -> None: |
| 483 | self.assertTrue(is_active_build({"state": "scheduled"})) |
| 484 | self.assertTrue(is_active_build({"state": "running"})) |
| 485 | self.assertTrue(is_active_build({"state": "waiting"})) |
| 486 | self.assertTrue(is_active_build({"blocked": True, "state": "passed"})) |
| 487 | self.assertFalse(is_active_build({"state": "failed"})) |
| 488 | |
| 489 | def test_build_payload_preserves_pr_context(self) -> None: |
| 490 | payload = create_build_payload( |
| 491 | actor="reviewer", |
| 492 | comment_id=99, |
| 493 | pr=make_pr(labels=[{"name": "ready"}, {"name": "v1"}]), |
| 494 | ) |
| 495 | self.assertEqual( |
| 496 | payload, |
| 497 | { |
| 498 | "commit": "0123456789abcdef", |
| 499 | "branch": "contributor:feature", |
| 500 | "message": "PR #42 /ci run by @reviewer", |
| 501 | "pull_request_id": 42, |
| 502 | "pull_request_base_branch": "main", |
| 503 | "pull_request_repository": ("https://github.com/contributor/vllm.git"), |
| 504 | "pull_request_labels": ["ready", "v1"], |
| 505 | "ignore_pipeline_branch_filters": True, |
| 506 | "env": { |
| 507 | "VLLM_CI_GITHUB_COMMENT_ID": "99", |
| 508 | "VLLM_CI_TRIGGERED_BY": "reviewer", |
| 509 | }, |
| 510 | "meta_data": { |
| 511 | "github-comment-id": "99", |
| 512 | "github-pr-number": "42", |
| 513 | "github-triggered-by": "reviewer", |
| 514 | }, |
| 515 | }, |
| 516 | ) |
| 517 | |
| 518 | def test_build_payload_uses_head_label_to_avoid_branch_collision(self) -> None: |
| 519 | pr = make_pr() |
| 520 | pr["head"]["ref"] = "main" |
| 521 | pr["head"]["label"] = "contributor:main" |
| 522 | |
| 523 | payload = create_build_payload(actor="reviewer", comment_id=99, pr=pr) |
| 524 | |
| 525 | self.assertEqual(payload["branch"], "contributor:main") |
| 526 | |
| 527 | def test_build_payload_falls_back_to_ref_without_label(self) -> None: |
| 528 | pr = make_pr() |
| 529 | del pr["head"]["label"] |
| 530 | |
| 531 | payload = create_build_payload(actor="reviewer", comment_id=99, pr=pr) |
| 532 | |
| 533 | self.assertEqual(payload["branch"], "feature") |
| 534 | |
| 535 | def test_write_reviewer_runs_ci_without_delegation(self) -> None: |
| 536 | github = FakeGitHub() |
| 537 | buildkite = FakeBuildkite([[], []]) |
| 538 | run(make_event(COMMAND_RUN_CI), github, buildkite) |
| 539 | |
| 540 | self.assertEqual(len(buildkite.created_builds), 1) |
| 541 | self.assertEqual( |
| 542 | buildkite.created_builds[0]["message"], |
| 543 | "PR #42 /ci run by @reviewer", |
| 544 | ) |
| 545 | self.assertEqual(github.reactions, ["eyes", "rocket"]) |
| 546 | self.assertTrue(github.comments[0].startswith("✅ ")) |
| 547 | self.assertIn("Buildkite CI #123", github.comments[0]) |
| 548 | |
| 549 | def test_run_variants_require_current_base_without_changing_commit(self) -> None: |
| 550 | for command in RUN_CI_COMMAND_ENV: |
| 551 | if command.endswith(" --allow-stale"): |
| 552 | continue |
| 553 | for behind in (0, 1, 5): |
| 554 | with self.subTest(command=command, behind=behind): |
| 555 | github = FakeGitHub( |
| 556 | behind=behind, pr=make_pr(base={"ref": "release/1.0"}) |
| 557 | ) |
| 558 | buildkite = FakeBuildkite([[], []]) |
| 559 | run(make_event(command), github, buildkite) |
| 560 | |
| 561 | self.assertEqual( |
| 562 | github.lag_queries, |
| 563 | [("release/1.0", github.pr["head"]["sha"])], |
| 564 | ) |
| 565 | if behind == 0: |
| 566 | self.assertEqual( |
| 567 | buildkite.created_builds[0]["commit"], |
| 568 | github.pr["head"]["sha"], |
| 569 | ) |
| 570 | self.assertIn( |
| 571 | "](https://buildkite.example/builds/123)", |
| 572 | github.comments[0], |
| 573 | ) |
| 574 | else: |
| 575 | self.assertEqual(buildkite.created_builds, []) |
| 576 | self.assertTrue(github.comments[0].startswith("❌ ")) |
| 577 | self.assertIn(str(behind), github.comments[0]) |
| 578 | self.assertIn("upstream `release/1.0`", github.comments[0]) |
| 579 | self.assertIn("every commit", github.comments[0]) |
| 580 | self.assertIn( |
| 581 | "No new CI build was started.", github.comments[0] |
| 582 | ) |
| 583 | self.assertIn(command, github.comments[0]) |
| 584 | |
| 585 | def test_allow_stale_preserves_author_permissions_and_run_configuration( |
| 586 | self, |
| 587 | ) -> None: |
| 588 | cases = ( |
| 589 | (COMMAND_RUN_CI, "ci", {}), |
| 590 | (COMMAND_RUN_CI_ALL, "ci", {"RUN_ALL": "1"}), |
| 591 | (COMMAND_RUN_CI_NIGHTLY, "ci", {"RUN_ALL": "1", "NIGHTLY": "1"}), |
| 592 | (COMMAND_RUN_AMD_CI, "amd-ci", {}), |
| 593 | (COMMAND_RUN_AMD_CI_ALL, "amd-ci", {"RUN_ALL": "1"}), |
| 594 | ( |
| 595 | COMMAND_RUN_AMD_CI_NIGHTLY, |
| 596 | "amd-ci", |
| 597 | {"RUN_ALL": "1", "NIGHTLY": "1"}, |
| 598 | ), |
| 599 | ) |
| 600 | for original_command, pipeline, expected_env in cases: |
| 601 | command = f"{original_command} --allow-stale" |
| 602 | for ready, behind in ((False, 100), (True, 0), (True, 100)): |
| 603 | with self.subTest(command=command, ready=ready, behind=behind): |
| 604 | github = FakeGitHub( |
| 605 | permission="read", |
| 606 | behind=behind, |
| 607 | pr=make_pr( |
| 608 | base={"ref": "feature/stack"}, |
| 609 | labels=[{"name": "ready"}] if ready else [], |
| 610 | ), |
| 611 | ) |
| 612 | buildkite = FakeBuildkite([[], []]) |
| 613 | |
| 614 | run(make_event(command, "author"), github, buildkite) |
| 615 | |
| 616 | self.assertEqual(pipeline_for_command(command), pipeline) |
| 617 | if not ready: |
| 618 | self.assertEqual(buildkite.created_builds, []) |
| 619 | self.assertEqual(buildkite.list_calls, []) |
| 620 | self.assertEqual(github.lag_queries, []) |
| 621 | self.assertTrue(github.comments[0].startswith("❌ ")) |
| 622 | continue |
| 623 | self.assertEqual(len(buildkite.created_builds), 1) |
| 624 | payload = buildkite.created_builds[0] |
| 625 | self.assertEqual(payload["commit"], github.pr["head"]["sha"]) |
| 626 | self.assertEqual( |
| 627 | payload["pull_request_base_branch"], "feature/stack" |
| 628 | ) |
| 629 | self.assertEqual(payload["message"], f"PR #42 {command} by @author") |
| 630 | self.assertEqual( |
| 631 | payload["env"], |
| 632 | { |
| 633 | "VLLM_CI_GITHUB_COMMENT_ID": "99", |
| 634 | "VLLM_CI_TRIGGERED_BY": "author", |
| 635 | **expected_env, |
| 636 | }, |
| 637 | ) |
| 638 | self.assertEqual( |
| 639 | github.lag_queries, |
| 640 | [("feature/stack", github.pr["head"]["sha"])], |
| 641 | ) |
| 642 | self.assertIn( |
| 643 | "Buildkite AMD CI #123" |
| 644 | if pipeline == "amd-ci" |
| 645 | else "Buildkite CI #123", |
| 646 | github.comments[0], |
| 647 | ) |
| 648 | if behind: |
| 649 | self.assertIn("100 commits behind", github.comments[0]) |
| 650 | self.assertIn("upstream `feature/stack`", github.comments[0]) |
| 651 | self.assertIn("Before merging", github.comments[0]) |
| 652 | self.assertIn("latest PR commit", github.comments[0]) |
| 653 | self.assertIn(original_command, github.comments[0]) |
| 654 | |
| 655 | def test_base_lag_query_qualifies_branch_refs_and_validates_count(self) -> None: |
| 656 | for base_ref, encoded_ref in ( |
| 657 | ("main", "main"), |
| 658 | ("release/1.0", "release%2F1.0"), |
| 659 | ("feature/stack", "feature%2Fstack"), |
| 660 | ): |
| 661 | for count in (0, 5, None, -1, True, "5"): |
| 662 | with self.subTest(base_ref=base_ref, count=count): |
| 663 | transport = FakeTransport( |
| 664 | {} if count is None else {"behind_by": count} |
| 665 | ) |
| 666 | github = GitHubClient("token", "vllm-project/vllm", transport) |
| 667 | if type(count) is int and count >= 0: |
| 668 | self.assertEqual( |
| 669 | github.get_commits_behind_base(base_ref, "head-sha"), count |
| 670 | ) |
| 671 | else: |
| 672 | with self.assertRaises(ApiError): |
| 673 | github.get_commits_behind_base(base_ref, "head-sha") |
| 674 | self.assertEqual( |
| 675 | transport.calls[0]["url"], |
| 676 | "https://api.github.com/repos/vllm-project/vllm/compare/" |
| 677 | f"refs%2Fheads%2F{encoded_ref}...head-sha?per_page=1", |
| 678 | ) |
| 679 | self.assertEqual(transport.calls[0]["method"], "GET") |
| 680 | |
| 681 | def test_unknown_lag_or_changed_pr_prevents_runs(self) -> None: |
| 682 | for command in ( |
| 683 | COMMAND_RUN_CI, |
| 684 | COMMAND_RUN_AMD_CI, |
| 685 | f"{COMMAND_RUN_CI} --allow-stale", |
| 686 | f"{COMMAND_RUN_AMD_CI} --allow-stale", |
| 687 | ): |
| 688 | for fault in ("api", "head", "closed", "base"): |
| 689 | with self.subTest(command=command, fault=fault): |
| 690 | github = FakeGitHub() |
| 691 | buildkite = FakeBuildkite([[], []]) |
| 692 | |
| 693 | def check( |
| 694 | base_ref: str, |
| 695 | sha: str, |
| 696 | fault: str = fault, |
| 697 | github: FakeGitHub = github, |
| 698 | ) -> int: |
| 699 | if fault == "api": |
| 700 | raise ApiError(503, "Unavailable") |
| 701 | changed = make_pr() |
| 702 | if fault == "head": |
| 703 | changed["head"]["sha"] = "another-head" |
| 704 | elif fault == "closed": |
| 705 | changed["state"] = "closed" |
| 706 | else: |
| 707 | changed["base"]["ref"] = "another-base" |
| 708 | github.prs[42] = changed |
| 709 | return 0 |
| 710 | |
| 711 | with patch.object( |
| 712 | github, "get_commits_behind_base", side_effect=check |
| 713 | ): |
| 714 | run(make_event(command), github, buildkite) |
| 715 | self.assertEqual(buildkite.created_builds, []) |
| 716 | self.assertEqual(buildkite.retry_calls, []) |
| 717 | self.assertTrue(github.comments[0].startswith("❌ ")) |
| 718 | self.assertNotIn("Triggered", github.comments[0]) |
| 719 | |
| 720 | def test_amd_run_ignores_blocked_builds(self) -> None: |
| 721 | for metadata in ({}, {"github-comment-id": "98"}): |
| 722 | with self.subTest(metadata=metadata): |
| 723 | github = FakeGitHub() |
| 724 | blocked_build = { |
| 725 | "blocked": True, |
| 726 | "created_at": "2026-08-18T01:00:00Z", |
| 727 | "meta_data": metadata, |
| 728 | "number": 122, |
| 729 | "pull_request": {"id": 42}, |
| 730 | "state": "blocked", |
| 731 | "web_url": "https://buildkite.example/amd-ci/builds/122", |
| 732 | } |
| 733 | buildkite = FakeBuildkite([[], [blocked_build]]) |
| 734 | |
| 735 | run(make_event(COMMAND_RUN_AMD_CI_ALL), github, buildkite) |
| 736 | |
| 737 | self.assertEqual(len(buildkite.created_builds), 1) |
| 738 | self.assertEqual(buildkite.created_builds[0]["env"]["RUN_ALL"], "1") |
| 739 | |
| 740 | def test_amd_run_deduplicates_comment_triggered_active_build(self) -> None: |
| 741 | github = FakeGitHub() |
| 742 | command_build = { |
| 743 | "blocked": False, |
| 744 | "created_at": "2026-08-18T01:00:00Z", |
| 745 | "meta_data": {"github-comment-id": "98"}, |
| 746 | "number": 122, |
| 747 | "pull_request": {"id": 42}, |
| 748 | "source": "api", |
| 749 | "state": "running", |
| 750 | "web_url": "https://buildkite.example/amd-ci/builds/122", |
| 751 | } |
| 752 | buildkite = FakeBuildkite([[], [command_build]]) |
| 753 | |
| 754 | run(make_event(COMMAND_RUN_AMD_CI_ALL), github, buildkite) |
| 755 | |
| 756 | self.assertEqual(buildkite.created_builds, []) |
| 757 | self.assertIn("AMD CI is already running", github.comments[0]) |
| 758 | |
| 759 | def test_run_all_sets_buildkite_environment(self) -> None: |
| 760 | github = FakeGitHub() |
| 761 | buildkite = FakeBuildkite([[], []]) |
| 762 | |
| 763 | run(make_event(COMMAND_RUN_CI_ALL), github, buildkite) |
| 764 | |
| 765 | payload = buildkite.created_builds[0] |
| 766 | self.assertEqual(payload["message"], "PR #42 /ci run all by @reviewer") |
| 767 | self.assertEqual(payload["env"]["RUN_ALL"], "1") |
| 768 | self.assertNotIn("NIGHTLY", payload["env"]) |
| 769 | |
| 770 | def test_run_nightly_sets_buildkite_environment(self) -> None: |
| 771 | github = FakeGitHub() |
| 772 | buildkite = FakeBuildkite([[], []]) |
| 773 | |
| 774 | run(make_event(COMMAND_RUN_CI_NIGHTLY), github, buildkite) |
| 775 | |
| 776 | payload = buildkite.created_builds[0] |
| 777 | self.assertEqual(payload["message"], "PR #42 /ci run nightly by @reviewer") |
| 778 | self.assertEqual(payload["env"]["RUN_ALL"], "1") |
| 779 | self.assertEqual(payload["env"]["NIGHTLY"], "1") |
| 780 | |
| 781 | def test_amd_run_variants_set_buildkite_environment(self) -> None: |
| 782 | cases = ( |
| 783 | (COMMAND_RUN_AMD_CI, {}), |
| 784 | (COMMAND_RUN_AMD_CI_ALL, {"RUN_ALL": "1"}), |
| 785 | ( |
| 786 | COMMAND_RUN_AMD_CI_NIGHTLY, |
| 787 | {"RUN_ALL": "1", "NIGHTLY": "1"}, |
| 788 | ), |
| 789 | ) |
| 790 | for command, expected_env in cases: |
| 791 | with self.subTest(command=command): |
| 792 | github = FakeGitHub() |
| 793 | buildkite = FakeBuildkite([[], []]) |
| 794 | |
| 795 | run(make_event(command), github, buildkite) |
| 796 | |
| 797 | payload = buildkite.created_builds[0] |
| 798 | self.assertEqual(payload["message"], f"PR #42 {command} by @reviewer") |
| 799 | command_env = { |
| 800 | key: value |
| 801 | for key, value in payload["env"].items() |
| 802 | if key in {"RUN_ALL", "NIGHTLY"} |
| 803 | } |
| 804 | self.assertEqual(command_env, expected_env) |
| 805 | self.assertIn("Buildkite AMD CI #123", github.comments[0]) |
| 806 | |
| 807 | def test_unapproved_authors_are_denied_without_buildkite(self) -> None: |
| 808 | github = FakeGitHub( |
| 809 | permission="read", |
| 810 | pr=make_pr(), |
| 811 | review_decision="REVIEW_REQUIRED", |
| 812 | ) |
| 813 | buildkite = FakeBuildkite() |
| 814 | run(make_event(COMMAND_RUN_CI, "author"), github, buildkite) |
| 815 | |
| 816 | self.assertEqual(buildkite.list_calls, []) |
| 817 | self.assertEqual(github.lag_queries, []) |
| 818 | self.assertEqual(github.reactions, ["eyes"]) |
| 819 | self.assertTrue(github.comments[0].startswith("❌ ")) |
| 820 | self.assertIn("approve the PR", github.comments[0]) |
| 821 | |
| 822 | run(make_event(COMMAND_RUN_CI, "author"), github, buildkite) |
| 823 | self.assertEqual(len(github.comments), 1) |
| 824 | |
| 825 | def test_unapproved_author_gets_amd_specific_guidance(self) -> None: |
| 826 | github = FakeGitHub( |
| 827 | permission="read", |
| 828 | pr=make_pr(), |
| 829 | review_decision="REVIEW_REQUIRED", |
| 830 | ) |
| 831 | buildkite = FakeBuildkite() |
| 832 | |
| 833 | run(make_event(COMMAND_RUN_AMD_CI, "author"), github, buildkite) |
| 834 | |
| 835 | self.assertEqual(buildkite.list_calls, []) |
| 836 | self.assertIn("`/amd-ci run`", github.comments[0]) |
| 837 | self.assertNotIn("`/ci run`", github.comments[0]) |
| 838 | |
| 839 | def test_untrusted_approval_cannot_launch_ci(self) -> None: |
| 840 | github = FakeGitHub( |
| 841 | permission="read", |
| 842 | pr=make_pr(), |
| 843 | review_decision="APPROVED", |
| 844 | reviews=[ |
| 845 | { |
| 846 | "state": "APPROVED", |
| 847 | "user": {"login": "untrusted-reviewer"}, |
| 848 | } |
| 849 | ], |
| 850 | ) |
| 851 | buildkite = FakeBuildkite() |
| 852 | |
| 853 | run(make_event(COMMAND_RUN_CI, "author"), github, buildkite) |
| 854 | |
| 855 | self.assertEqual(buildkite.list_calls, []) |
| 856 | self.assertEqual(github.reactions, ["eyes"]) |
| 857 | self.assertTrue(github.comments[0].startswith("❌ ")) |
| 858 | |
| 859 | def test_ready_label_notifies_author_once(self) -> None: |
| 860 | pr = make_pr(labels=[{"name": "ready"}]) |
| 861 | event = { |
| 862 | "action": "labeled", |
| 863 | "label": {"name": "ready"}, |
| 864 | "pull_request": pr, |
| 865 | } |
| 866 | github = FakeGitHub(permission="read", pr=pr) |
| 867 | |
| 868 | notify_authorized(event, github) |
| 869 | notify_authorized(event, github) |
| 870 | |
| 871 | self.assertEqual(len(github.comments), 1) |
| 872 | comment = github.comments[0] |
| 873 | self.assertTrue(comment.startswith("✅ @author")) |
| 874 | self.assertIn("`/ci run` starts upstream CI", comment) |
| 875 | self.assertIn("`/ci retry` retries failed jobs", comment) |
| 876 | self.assertIn("`/ci cancel` cancels scheduled or running", comment) |
| 877 | self.assertIn("`/amd-ci run` starts AMD CI only", comment) |
| 878 | self.assertIn("`/amd-ci retry` retries failed jobs in AMD CI", comment) |
| 879 | self.assertIn("`/amd-ci cancel` does the same for AMD CI only", comment) |
| 880 | self.assertIn("CI build for the current PR head", comment) |
| 881 | self.assertIn("only jobs that failed in the latest earlier CI build", comment) |
| 882 | self.assertNotIn(COMMAND_RUN_CI_ALL, comment) |
| 883 | self.assertNotIn(COMMAND_RUN_CI_NIGHTLY, comment) |
| 884 | self.assertIn(CI_AUTHORIZED_COMMENT_MARKER, comment) |
| 885 | |
| 886 | def test_ready_label_notifies_after_missed_approval_notification(self) -> None: |
| 887 | pr = make_pr(labels=[{"name": "ready"}]) |
| 888 | event = { |
| 889 | "action": "labeled", |
| 890 | "label": {"name": "ready"}, |
| 891 | "pull_request": pr, |
| 892 | } |
| 893 | github = FakeGitHub( |
| 894 | permission="read", |
| 895 | permissions={"reviewer": "write"}, |
| 896 | pr=pr, |
| 897 | review_decision="APPROVED", |
| 898 | reviews=[ |
| 899 | { |
| 900 | "state": "APPROVED", |
| 901 | "user": {"login": "reviewer"}, |
| 902 | } |
| 903 | ], |
| 904 | ) |
| 905 | |
| 906 | notify_authorized(event, github) |
| 907 | |
| 908 | self.assertEqual(len(github.comments), 1) |
| 909 | self.assertIn("@author", github.comments[0]) |
| 910 | |
| 911 | def test_trusted_approval_notifies_author_once(self) -> None: |
| 912 | event = { |
| 913 | "action": "submitted", |
| 914 | "pull_request": make_pr(), |
| 915 | "review": {"state": "approved"}, |
| 916 | } |
| 917 | github = FakeGitHub( |
| 918 | permission="read", |
| 919 | permissions={"reviewer": "write"}, |
| 920 | review_decision="APPROVED", |
| 921 | reviews=[ |
| 922 | { |
| 923 | "state": "APPROVED", |
| 924 | "user": {"login": "reviewer"}, |
| 925 | } |
| 926 | ], |
| 927 | ) |
| 928 | |
| 929 | notify_authorized(event, github) |
| 930 | notify_authorized(event, github) |
| 931 | |
| 932 | self.assertEqual(len(github.comments), 1) |
| 933 | self.assertIn("@author", github.comments[0]) |
| 934 | |
| 935 | def test_workflow_run_resolves_pr_from_head_commit(self) -> None: |
| 936 | pr = make_pr() |
| 937 | github = FakeGitHub( |
| 938 | pr=pr, |
| 939 | pulls_for_commit=[{"number": pr["number"]}], |
| 940 | ) |
| 941 | |
| 942 | resolved = resolve_workflow_run_pr( |
| 943 | {"head_sha": pr["head"]["sha"], "pull_requests": []}, |
| 944 | github, |
| 945 | ) |
| 946 | |
| 947 | self.assertEqual(resolved, pr) |
| 948 | |
| 949 | def test_workflow_run_ignores_unrelated_pr_association(self) -> None: |
| 950 | pr = make_pr() |
| 951 | github = FakeGitHub( |
| 952 | pr=pr, |
| 953 | pulls_for_commit=[{"number": pr["number"]}], |
| 954 | ) |
| 955 | |
| 956 | resolved = resolve_workflow_run_pr( |
| 957 | { |
| 958 | "head_sha": pr["head"]["sha"], |
| 959 | "pull_requests": [{"number": 1}], |
| 960 | }, |
| 961 | github, |
| 962 | ) |
| 963 | |
| 964 | self.assertEqual(resolved, pr) |
| 965 | |
| 966 | def test_approval_does_not_notify_when_ready_label_exists(self) -> None: |
| 967 | pr = make_pr(labels=[{"name": "ready"}]) |
| 968 | event = { |
| 969 | "action": "submitted", |
| 970 | "pull_request": pr, |
| 971 | "review": {"state": "approved"}, |
| 972 | } |
| 973 | github = FakeGitHub( |
| 974 | permission="read", |
| 975 | permissions={"reviewer": "write"}, |
| 976 | pr=pr, |
| 977 | review_decision="APPROVED", |
| 978 | reviews=[ |
| 979 | { |
| 980 | "state": "APPROVED", |
| 981 | "user": {"login": "reviewer"}, |
| 982 | } |
| 983 | ], |
| 984 | ) |
| 985 | |
| 986 | notify_authorized(event, github) |
| 987 | |
| 988 | self.assertEqual(github.comments, []) |
| 989 | |
| 990 | def test_untrusted_approval_does_not_notify_author(self) -> None: |
| 991 | event = { |
| 992 | "action": "submitted", |
| 993 | "pull_request": make_pr(), |
| 994 | "review": {"state": "approved"}, |
| 995 | } |
| 996 | github = FakeGitHub( |
| 997 | permission="read", |
| 998 | review_decision="APPROVED", |
| 999 | reviews=[ |
| 1000 | { |
| 1001 | "state": "APPROVED", |
| 1002 | "user": {"login": "reviewer"}, |
| 1003 | } |
| 1004 | ], |
| 1005 | ) |
| 1006 | |
| 1007 | notify_authorized(event, github) |
| 1008 | |
| 1009 | self.assertEqual(github.comments, []) |
| 1010 | |
| 1011 | def test_notification_skips_authors_who_already_have_write(self) -> None: |
| 1012 | pr = make_pr(labels=[{"name": "ready"}]) |
| 1013 | event = { |
| 1014 | "action": "labeled", |
| 1015 | "label": {"name": "ready"}, |
| 1016 | "pull_request": pr, |
| 1017 | } |
| 1018 | github = FakeGitHub(permission="write", pr=pr) |
| 1019 | |
| 1020 | notify_authorized(event, github) |
| 1021 | |
| 1022 | self.assertEqual(github.comments, []) |
| 1023 | |
| 1024 | def test_notification_skips_draft_prs(self) -> None: |
| 1025 | pr = make_pr(draft=True, labels=[{"name": "ready"}]) |
| 1026 | event = { |
| 1027 | "action": "labeled", |
| 1028 | "label": {"name": "ready"}, |
| 1029 | "pull_request": pr, |
| 1030 | } |
| 1031 | github = FakeGitHub(permission="read", pr=pr) |
| 1032 | |
| 1033 | notify_authorized(event, github) |
| 1034 | |
| 1035 | self.assertEqual(github.comments, []) |
| 1036 | |
| 1037 | def test_ci_retry_retries_failed_jobs_while_build_is_running(self) -> None: |
| 1038 | github = FakeGitHub( |
| 1039 | behind=100, |
| 1040 | permission="read", |
| 1041 | pr=make_pr(labels=[{"name": "ready"}]), |
| 1042 | ) |
| 1043 | buildkite = FakeBuildkite( |
| 1044 | [ |
| 1045 | [ |
| 1046 | { |
| 1047 | "created_at": "2026-07-28T01:00:00Z", |
| 1048 | "number": 123, |
| 1049 | "pull_request": {"id": 42}, |
| 1050 | "state": "failing", |
| 1051 | "web_url": "https://buildkite.example/builds/123", |
| 1052 | } |
| 1053 | ] |
| 1054 | ] |
| 1055 | ) |
| 1056 | run(make_event(COMMAND_RETRY_FAILED, "author"), github, buildkite) |
| 1057 | |
| 1058 | self.assertEqual(buildkite.retry_calls, [(123, RETRY_STATES)]) |
| 1059 | self.assertEqual(github.lag_queries, []) |
| 1060 | self.assertIn("Queued 3 failed job", github.comments[0]) |
| 1061 | |
| 1062 | def test_amd_ci_retry_retries_only_the_current_head_build(self) -> None: |
| 1063 | github = FakeGitHub( |
| 1064 | behind=100, |
| 1065 | permission="read", |
| 1066 | pr=make_pr(labels=[{"name": "ready"}]), |
| 1067 | ) |
| 1068 | buildkite = FakeBuildkite( |
| 1069 | [ |
| 1070 | [ |
| 1071 | { |
| 1072 | "created_at": "2026-07-28T01:00:00Z", |
| 1073 | "number": 321, |
| 1074 | "pull_request": {"id": 42}, |
| 1075 | "state": "failing", |
| 1076 | "web_url": "https://buildkite.example/amd-ci/builds/321", |
| 1077 | } |
| 1078 | ] |
| 1079 | ] |
| 1080 | ) |
| 1081 | |
| 1082 | run(make_event(COMMAND_RETRY_AMD_FAILED, "author"), github, buildkite) |
| 1083 | |
| 1084 | self.assertEqual(buildkite.retry_calls, [(321, RETRY_STATES)]) |
| 1085 | self.assertEqual(github.lag_queries, []) |
| 1086 | self.assertIn("Buildkite AMD CI #321", github.comments[0]) |
| 1087 | |
| 1088 | def test_amd_ci_retry_requires_a_build_for_the_current_head(self) -> None: |
| 1089 | github = FakeGitHub( |
| 1090 | permission="read", |
| 1091 | pr=make_pr(labels=[{"name": "ready"}]), |
| 1092 | ) |
| 1093 | buildkite = FakeBuildkite([[]]) |
| 1094 | |
| 1095 | run(make_event(COMMAND_RETRY_AMD_FAILED, "author"), github, buildkite) |
| 1096 | |
| 1097 | self.assertEqual(buildkite.list_calls, [("0123456789abcdef", None)]) |
| 1098 | self.assertEqual(buildkite.retry_calls, []) |
| 1099 | self.assertEqual(buildkite.created_builds, []) |
| 1100 | self.assertIn( |
| 1101 | "No AMD CI build exists for the current PR head", github.comments[0] |
| 1102 | ) |
| 1103 | self.assertIn("Use `/amd-ci run`", github.comments[0]) |
| 1104 | |
| 1105 | def test_ci_retry_creates_filtered_build_for_new_head(self) -> None: |
| 1106 | github = FakeGitHub( |
| 1107 | behind=100, |
| 1108 | permission="read", |
| 1109 | pr=make_pr(labels=[{"name": "ready"}]), |
| 1110 | ) |
| 1111 | source_build = { |
| 1112 | "commit": "old-commit", |
| 1113 | "created_at": "2026-07-28T01:00:00Z", |
| 1114 | "finished_at": "2026-07-28T02:00:00Z", |
| 1115 | "number": 122, |
| 1116 | "pull_request": {"id": 42}, |
| 1117 | "state": "failed", |
| 1118 | "web_url": "https://buildkite.example/builds/122", |
| 1119 | } |
| 1120 | buildkite = FakeBuildkite( |
| 1121 | [[], [source_build]], |
| 1122 | [ |
| 1123 | [ |
| 1124 | { |
| 1125 | "state": "failed", |
| 1126 | "step_key": "basic-models-test-other-cpu", |
| 1127 | "type": "script", |
| 1128 | }, |
| 1129 | { |
| 1130 | "state": "failed", |
| 1131 | "step_key": "basic-models-test-other-cpu", |
| 1132 | "type": "script", |
| 1133 | }, |
| 1134 | { |
| 1135 | "state": "timed_out", |
| 1136 | "step_key": "distributed-tests-2xh100-2xmi300", |
| 1137 | "type": "script", |
| 1138 | }, |
| 1139 | ] |
| 1140 | ], |
| 1141 | ) |
| 1142 | |
| 1143 | run(make_event(COMMAND_RETRY_FAILED, "author"), github, buildkite) |
| 1144 | |
| 1145 | self.assertEqual(buildkite.job_list_calls, [122]) |
| 1146 | self.assertEqual(len(buildkite.created_builds), 1) |
| 1147 | payload = buildkite.created_builds[0] |
| 1148 | self.assertEqual(payload["commit"], "0123456789abcdef") |
| 1149 | self.assertEqual(github.lag_queries, []) |
| 1150 | self.assertEqual(payload["message"], "PR #42 /ci retry by @author") |
| 1151 | self.assertEqual( |
| 1152 | json.loads(payload["env"]["VLLM_CI_ONLY_STEP_KEYS"]), |
| 1153 | [ |
| 1154 | "basic-models-test-other-cpu", |
| 1155 | "distributed-tests-2xh100-2xmi300", |
| 1156 | ], |
| 1157 | ) |
| 1158 | self.assertEqual( |
| 1159 | payload["meta_data"]["github-retry-source-build"], |
| 1160 | "122", |
| 1161 | ) |
| 1162 | self.assertEqual( |
| 1163 | payload["meta_data"]["github-retry-source-commit"], |
| 1164 | "old-commit", |
| 1165 | ) |
| 1166 | self.assertIn("running 2 failed step", github.comments[0]) |
| 1167 | self.assertIn("Buildkite CI #122", github.comments[0]) |
| 1168 | |
| 1169 | def test_ci_retry_new_head_requires_stable_step_keys(self) -> None: |
| 1170 | github = FakeGitHub( |
| 1171 | permission="read", |
| 1172 | pr=make_pr(labels=[{"name": "ready"}]), |
| 1173 | ) |
| 1174 | buildkite = FakeBuildkite( |
| 1175 | [ |
| 1176 | [], |
| 1177 | [ |
| 1178 | { |
| 1179 | "commit": "old-commit", |
| 1180 | "created_at": "2026-07-28T01:00:00Z", |
| 1181 | "finished_at": "2026-07-28T02:00:00Z", |
| 1182 | "number": 122, |
| 1183 | "pull_request": {"id": 42}, |
| 1184 | "state": "failed", |
| 1185 | "web_url": "https://buildkite.example/builds/122", |
| 1186 | } |
| 1187 | ], |
| 1188 | ], |
| 1189 | [[{"state": "failed", "step_key": None, "type": "script"}]], |
| 1190 | ) |
| 1191 | |
| 1192 | run(make_event(COMMAND_RETRY_FAILED, "author"), github, buildkite) |
| 1193 | |
| 1194 | self.assertEqual(buildkite.created_builds, []) |
| 1195 | self.assertIn("without stable step keys", github.comments[0]) |
| 1196 | self.assertIn("Use `/ci run`", github.comments[0]) |
| 1197 | |
| 1198 | def test_ci_retry_new_head_rejects_incomplete_setup_failure(self) -> None: |
| 1199 | github = FakeGitHub( |
| 1200 | permission="read", |
| 1201 | pr=make_pr(labels=[{"name": "ready"}]), |
| 1202 | ) |
| 1203 | buildkite = FakeBuildkite( |
| 1204 | [ |
| 1205 | [], |
| 1206 | [ |
| 1207 | { |
| 1208 | "commit": "old-commit", |
| 1209 | "created_at": "2026-07-28T01:00:00Z", |
| 1210 | "finished_at": "2026-07-28T02:00:00Z", |
| 1211 | "number": 122, |
| 1212 | "pull_request": {"id": 42}, |
| 1213 | "state": "failed", |
| 1214 | "web_url": "https://buildkite.example/builds/122", |
| 1215 | } |
| 1216 | ], |
| 1217 | ], |
| 1218 | [ |
| 1219 | [ |
| 1220 | { |
| 1221 | "state": "failed", |
| 1222 | "step_key": "image-build", |
| 1223 | "type": "script", |
| 1224 | } |
| 1225 | ] |
| 1226 | ], |
| 1227 | ) |
| 1228 | |
| 1229 | run(make_event(COMMAND_RETRY_FAILED, "author"), github, buildkite) |
| 1230 | |
| 1231 | self.assertEqual(buildkite.created_builds, []) |
| 1232 | self.assertIn("failed during CI setup", github.comments[0]) |
| 1233 | self.assertIn("Use `/ci run`", github.comments[0]) |
| 1234 | |
| 1235 | def test_buildkite_retry_uses_retry_failed_jobs_endpoint(self) -> None: |
| 1236 | transport = FakeTransport({"retried_jobs_count": 2}) |
| 1237 | client = BuildkiteClient( |
| 1238 | "secret", |
| 1239 | "vllm", |
| 1240 | "ci", |
| 1241 | transport=transport, |
| 1242 | ) |
| 1243 | client.retry_failed_jobs(123, RETRY_STATES) |
| 1244 | |
| 1245 | call = transport.calls[0] |
| 1246 | self.assertEqual(call["method"], "PUT") |
| 1247 | self.assertTrue(call["url"].endswith("/123/retry_failed_jobs")) |
| 1248 | self.assertEqual(call["body"], {"states": RETRY_STATES}) |
| 1249 | |
| 1250 | def test_ci_cancel_cancels_active_builds_for_pr_branch(self) -> None: |
| 1251 | github = FakeGitHub() |
| 1252 | buildkite = FakeBuildkite( |
| 1253 | [ |
| 1254 | # Builds created under the head label (owner:branch). |
| 1255 | [ |
| 1256 | { |
| 1257 | "branch": "contributor:feature", |
| 1258 | "number": 123, |
| 1259 | "pull_request": {"id": 42}, |
| 1260 | "state": "running", |
| 1261 | "web_url": "https://buildkite.example/builds/123", |
| 1262 | }, |
| 1263 | { |
| 1264 | "branch": "contributor:feature", |
| 1265 | "number": 124, |
| 1266 | "meta_data": {"github-pr-number": "42"}, |
| 1267 | "state": "failing", |
| 1268 | "web_url": "https://buildkite.example/builds/124", |
| 1269 | }, |
| 1270 | { |
| 1271 | "branch": "contributor:feature", |
| 1272 | "number": 125, |
| 1273 | "pull_request": {"id": 43}, |
| 1274 | "state": "running", |
| 1275 | "web_url": "https://buildkite.example/builds/125", |
| 1276 | }, |
| 1277 | { |
| 1278 | "branch": "contributor:feature", |
| 1279 | "number": 127, |
| 1280 | "pull_request": {"id": 42}, |
| 1281 | "state": "passed", |
| 1282 | "web_url": "https://buildkite.example/builds/127", |
| 1283 | }, |
| 1284 | ], |
| 1285 | # Legacy builds created under the bare ref before the label change. |
| 1286 | [ |
| 1287 | { |
| 1288 | "branch": "feature", |
| 1289 | "number": 126, |
| 1290 | "pull_request": {"id": 42}, |
| 1291 | "state": "running", |
| 1292 | "web_url": "https://buildkite.example/builds/126", |
| 1293 | }, |
| 1294 | ], |
| 1295 | ] |
| 1296 | ) |
| 1297 | |
| 1298 | run(make_event(COMMAND_CANCEL_CI), github, buildkite) |
| 1299 | |
| 1300 | self.assertEqual(buildkite.cancel_calls, [123, 124, 126]) |
| 1301 | self.assertEqual(github.lag_queries, []) |
| 1302 | self.assertEqual( |
| 1303 | [request["branch"] for request in buildkite.list_requests], |
| 1304 | ["contributor:feature", "feature"], |
| 1305 | ) |
| 1306 | self.assertIn("Requested cancellation of 3 CI builds", github.comments[0]) |
| 1307 | self.assertIn("#123", github.comments[0]) |
| 1308 | self.assertIn("#124", github.comments[0]) |
| 1309 | self.assertIn("#126", github.comments[0]) |
| 1310 | |
| 1311 | def test_ci_cancel_is_a_noop_without_active_builds(self) -> None: |
| 1312 | github = FakeGitHub() |
| 1313 | buildkite = FakeBuildkite([[], []]) |
| 1314 | |
| 1315 | run(make_event(COMMAND_CANCEL_CI), github, buildkite) |
| 1316 | |
| 1317 | self.assertEqual(buildkite.cancel_calls, []) |
| 1318 | self.assertIn("No cancelable CI build is running", github.comments[0]) |
| 1319 | |
| 1320 | def test_amd_ci_cancel_handles_command_and_fork_webhook_branches(self) -> None: |
| 1321 | pr = make_pr() |
| 1322 | pr["head"]["label"] = "contributor:feature" |
| 1323 | github = FakeGitHub(pr=pr) |
| 1324 | buildkite = FakeBuildkite( |
| 1325 | [ |
| 1326 | [ |
| 1327 | { |
| 1328 | "branch": "contributor:feature", |
| 1329 | "number": 322, |
| 1330 | "pull_request": {"id": 42}, |
| 1331 | "state": "failing", |
| 1332 | "web_url": "https://buildkite.example/amd-ci/builds/322", |
| 1333 | } |
| 1334 | ], |
| 1335 | [ |
| 1336 | { |
| 1337 | "branch": "feature", |
| 1338 | "number": 321, |
| 1339 | "pull_request": {"id": 42}, |
| 1340 | "state": "running", |
| 1341 | "web_url": "https://buildkite.example/amd-ci/builds/321", |
| 1342 | } |
| 1343 | ], |
| 1344 | ] |
| 1345 | ) |
| 1346 | |
| 1347 | run(make_event(COMMAND_CANCEL_AMD_CI), github, buildkite) |
| 1348 | |
| 1349 | self.assertEqual(buildkite.cancel_calls, [322, 321]) |
| 1350 | self.assertEqual(github.lag_queries, []) |
| 1351 | self.assertEqual( |
| 1352 | [request["branch"] for request in buildkite.list_requests], |
| 1353 | ["contributor:feature", "feature"], |
| 1354 | ) |
| 1355 | self.assertTrue( |
| 1356 | all( |
| 1357 | request["states"] == CANCELABLE_BUILD_STATES |
| 1358 | for request in buildkite.list_requests |
| 1359 | ) |
| 1360 | ) |
| 1361 | self.assertIn("cancellation of 2 AMD CI builds", github.comments[0]) |
| 1362 | |
| 1363 | def test_buildkite_cancel_uses_cancel_build_endpoint(self) -> None: |
| 1364 | transport = FakeTransport({"number": 123, "state": "canceling"}) |
| 1365 | client = BuildkiteClient( |
| 1366 | "secret", |
| 1367 | "vllm", |
| 1368 | "ci", |
| 1369 | transport=transport, |
| 1370 | ) |
| 1371 | |
| 1372 | client.cancel_build(123) |
| 1373 | |
| 1374 | call = transport.calls[0] |
| 1375 | self.assertEqual(call["method"], "PUT") |
| 1376 | self.assertTrue(call["url"].endswith("/123/cancel")) |
| 1377 | self.assertIsNone(call["body"]) |
| 1378 | |
| 1379 | def test_buildkite_list_builds_allows_query_on_builds_endpoint(self) -> None: |
| 1380 | transport = FakeTransport([]) |
| 1381 | client = BuildkiteClient( |
| 1382 | "secret", |
| 1383 | "vllm", |
| 1384 | "ci", |
| 1385 | transport=transport, |
| 1386 | ) |
| 1387 | |
| 1388 | builds = client.list_builds( |
| 1389 | "current-commit", |
| 1390 | branch="feature", |
| 1391 | metadata=("github-pr-number", "42"), |
| 1392 | states=CANCELABLE_BUILD_STATES, |
| 1393 | ) |
| 1394 | |
| 1395 | self.assertEqual(builds, []) |
| 1396 | url = transport.calls[0]["url"] |
| 1397 | self.assertIn("?exclude_jobs=true", url) |
| 1398 | self.assertIn("commit=current-commit", url) |
| 1399 | self.assertIn("branch=feature", url) |
| 1400 | self.assertIn("meta_data%5Bgithub-pr-number%5D=42", url) |
| 1401 | self.assertIn("state%5B%5D=scheduled", url) |
| 1402 | self.assertIn("state%5B%5D=running", url) |
| 1403 | self.assertIn("state%5B%5D=failing", url) |
| 1404 | |
| 1405 | def test_buildkite_failed_jobs_follow_cursor_pagination(self) -> None: |
| 1406 | next_url = ( |
| 1407 | "https://api.buildkite.com/v2/organizations/vllm/pipelines/ci/" |
| 1408 | "builds/123/jobs?after=cursor" |
| 1409 | ) |
| 1410 | transport = FakeTransport( |
| 1411 | ( |
| 1412 | { |
| 1413 | "items": [{"id": "first"}], |
| 1414 | "links": {"next": next_url}, |
| 1415 | }, |
| 1416 | { |
| 1417 | "items": [{"id": "second"}], |
| 1418 | "links": {"next": None}, |
| 1419 | }, |
| 1420 | ) |
| 1421 | ) |
| 1422 | client = BuildkiteClient( |
| 1423 | "secret", |
| 1424 | "vllm", |
| 1425 | "ci", |
| 1426 | transport=transport, |
| 1427 | ) |
| 1428 | |
| 1429 | jobs = client.list_failed_jobs(123) |
| 1430 | |
| 1431 | self.assertEqual(jobs, [{"id": "first"}, {"id": "second"}]) |
| 1432 | self.assertIn("state%5B%5D=failed", transport.calls[0]["url"]) |
| 1433 | self.assertIn("include_retried_jobs=false", transport.calls[0]["url"]) |
| 1434 | self.assertEqual(transport.calls[1]["url"], next_url) |
| 1435 | |
| 1436 | |
| 1437 | if __name__ == "__main__": |
| 1438 | unittest.main() |