[CI] Improve comment-triggered authorization and retries (#50414)
Signed-off-by: khluu <khluu000@gmail.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
Kevin H. Luu committed
Jul 30, 2026 at 02:48 UTC
48a077e4cfaa5425ac5df67ce95f07a99c6d26d5
4 files changed
+363
-17
.github/workflows/notify-ci-authorized.yml
new
+49
@@ -0,0 +1,49 @@
1
+name: Notify CI authorization
2
+
3
+on:
4
+ pull_request_target:
5
+ types: [labeled]
6
+ workflow_run:
7
+ workflows: [Record CI approval]
8
+ types: [completed]
9
+
10
+concurrency:
11
+ group: >-
12
+ notify-ci-authorized-${{
13
+ github.event.pull_request.number ||
14
+ github.event.workflow_run.pull_requests[0].number
15
+ }}
16
+ cancel-in-progress: false
17
+
18
+permissions:
19
+ contents: read
20
+ issues: write
21
+ pull-requests: read
22
+
23
+jobs:
24
+ notify:
25
+ if: >-
26
+ (github.event_name == 'pull_request_target' &&
27
+ github.event.action == 'labeled' &&
28
+ (github.event.label.name == 'ready' ||
29
+ github.event.label.name == 'ready-run-all-tests')) ||
30
+ (github.event_name == 'workflow_run' &&
31
+ github.event.workflow_run.event == 'pull_request_review' &&
32
+ github.event.workflow_run.conclusion == 'success' &&
33
+ github.event.workflow_run.pull_requests[0])
34
+ runs-on: ubuntu-latest
35
+ steps:
36
+ - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
37
+ with:
38
+ ref: ${{ github.event.repository.default_branch }}
39
+ persist-credentials: false
40
+ - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
41
+ with:
42
+ python-version: "3.12"
43
+ - name: Notify PR author that CI is available
44
+ run: >-
45
+ uv run --no-project --python 3.12
46
+ .github/workflows/scripts/run_ci_command.py
47
+ env:
48
+ CI_TRUSTED_USERS: ${{ vars.CI_TRUSTED_USERS }}
49
+ GH_TOKEN: ${{ github.token }}
.github/workflows/record-ci-approval.yml
new
+14
@@ -0,0 +1,14 @@
1
+name: Record CI approval
2
+
3
+on:
4
+ pull_request_review:
5
+ types: [submitted]
6
+
7
+permissions: {}
8
+
9
+jobs:
10
+ approved:
11
+ if: github.event.review.state == 'approved'
12
+ runs-on: ubuntu-latest
13
+ steps:
14
+ - run: echo "Approval recorded"
.github/workflows/scripts/run_ci_command.py
+121
-11
@@ -12,6 +12,7 @@ from typing import Any
12
13
COMMAND_RUN_CI = "/ci run"
14
COMMAND_RETRY_FAILED = "/ci retry"
15
+CI_AUTHORIZED_COMMENT_MARKER = "<!-- vllm-ci-authorized -->"
16
READY_LABELS = {"ready", "ready-run-all-tests"}
17
TRUSTED_PERMISSIONS = {"admin", "maintain", "write"}
18
ACTIVE_BUILD_STATES = {
@@ -175,6 +176,9 @@ class GitHubClient:
176
def list_reviews(self, number: int) -> list[dict[str, Any]]:
177
return self._paginate(self._repo_path(f"/pulls/{number}/reviews"))
178
179
+ def list_issue_comments(self, number: int) -> list[dict[str, Any]]:
180
+ return self._paginate(self._repo_path(f"/issues/{number}/comments"))
181
+
182
def list_reactions(self, comment_id: int) -> list[dict[str, Any]]:
183
return self._paginate(
184
self._repo_path(f"/issues/comments/{comment_id}/reactions")
@@ -484,12 +488,90 @@ def add_reaction_safely(
488
print(f"Could not add {content} reaction: {error}", file=sys.stderr)
489
490
487
-def is_already_handled(github: GitHubClient, comment_id: int) -> bool:
491
+def command_comment_marker(comment_id: int) -> str:
492
+ return f"<!-- vllm-ci-command:{comment_id} -->"
493
+
494
+
495
+def has_bot_comment_marker(
496
+ github: GitHubClient,
497
+ issue_number: int,
498
+ marker: str,
499
+) -> bool:
500
return any(
501
+ marker in str(comment.get("body", ""))
502
+ and (comment.get("user") or {}).get("login") == "github-actions[bot]"
503
+ for comment in github.list_issue_comments(issue_number)
504
+ )
505
+
506
+
507
+def is_already_handled(
508
+ github: GitHubClient,
509
+ issue_number: int,
510
+ comment_id: int,
511
+) -> bool:
512
+ terminal_reaction = any(
513
reaction.get("content") in {"rocket", "-1"}
514
and (reaction.get("user") or {}).get("login") == "github-actions[bot]"
515
for reaction in github.list_reactions(comment_id)
516
)
517
+ if terminal_reaction:
518
+ return True
519
+ return has_bot_comment_marker(
520
+ github,
521
+ issue_number,
522
+ command_comment_marker(comment_id),
523
+ )
524
+
525
+
526
+def notify_authorized(
527
+ event: Mapping[str, Any],
528
+ github: GitHubClient,
529
+ trusted_users_value: str = "",
530
+) -> None:
531
+ pr = event["pull_request"]
532
+ if pr["state"] != "open" or pr["draft"]:
533
+ return
534
+
535
+ trusted_users = parse_trusted_users(trusted_users_value)
536
+ author = pr["user"]["login"]
537
+ author_permission = github.get_permission(author)
538
+ if (
539
+ is_trusted_permission(author_permission)
540
+ or author.casefold() in trusted_users
541
+ or has_bot_comment_marker(
542
+ github,
543
+ pr["number"],
544
+ CI_AUTHORIZED_COMMENT_MARKER,
545
+ )
546
+ ):
547
+ return
548
+
549
+ if "label" in event:
550
+ if (
551
+ event.get("action") != "labeled"
552
+ or event["label"]["name"] not in READY_LABELS
553
+ or has_trusted_approval(github, pr["number"], trusted_users)
554
+ ):
555
+ return
556
+ elif "review" in event:
557
+ if (
558
+ event.get("action") != "submitted"
559
+ or str(event["review"].get("state", "")).casefold() != "approved"
560
+ or has_ready_label(pr)
561
+ or not has_trusted_approval(github, pr["number"], trusted_users)
562
+ ):
563
+ return
564
+ else:
565
+ return
566
+
567
+ github.add_comment(
568
+ pr["number"],
569
+ (
570
+ f"✅ @{author}, CI is now available for this PR. Comment `/ci run` "
571
+ "to run full CI or `/ci retry` to retry failed jobs.\n\n"
572
+ f"{CI_AUTHORIZED_COMMENT_MARKER}"
573
+ ),
574
+ )
575
576
577
def handle_run_ci(
@@ -553,8 +635,6 @@ def handle_retry_failed(
635
metadata = build.get("meta_data") or {}
636
if str(metadata.get("github-comment-id")) == str(comment_id):
637
return f"CI was already requested by this comment: {build['web_url']}"
556
- if not build.get("finished_at") or is_active_build(build):
557
- return f"CI is still running for this commit: {build['web_url']}"
638
639
retried = buildkite.retry_failed_jobs(build["number"], RETRY_STATES)
640
if retried["retried_jobs_count"] == 0:
@@ -652,7 +732,7 @@ def run(
732
comment_id = event["comment"]["id"]
733
actor = event["comment"]["user"]["login"]
734
655
- if is_already_handled(github, comment_id):
735
+ if is_already_handled(github, issue_number, comment_id):
736
print(f"Comment {comment_id} was already handled.")
737
return
738
add_reaction_safely(github, comment_id, "eyes")
@@ -661,7 +741,11 @@ def run(
741
pr = github.get_pr(issue_number)
742
permission = github.get_permission(actor)
743
if pr["state"] != "open":
664
- github.add_comment(issue_number, "CI commands require an open PR.")
744
+ github.add_comment(
745
+ issue_number,
746
+ "❌ CI commands require an open PR.\n\n"
747
+ f"{command_comment_marker(comment_id)}",
748
+ )
749
return
750
751
trusted_users = parse_trusted_users(trusted_users_value)
@@ -685,8 +769,10 @@ def run(
769
trusted_users=trusted_users,
770
)
771
if not allowed:
688
- add_reaction_safely(github, comment_id, "-1")
689
- github.add_comment(issue_number, f"@{actor}, {reason}")
772
+ github.add_comment(
773
+ issue_number,
774
+ f"❌ @{actor}, {reason}\n\n{command_comment_marker(comment_id)}",
775
+ )
776
return
777
778
print(f"Authorized @{actor}: {reason}")
@@ -707,7 +793,7 @@ def run(
793
pr=pr,
794
)
795
add_reaction_safely(github, comment_id, "rocket")
710
- github.add_comment(issue_number, message)
796
+ github.add_comment(issue_number, f"✅ {message}")
797
except Exception:
798
add_reaction_safely(github, comment_id, "confused")
799
raise
@@ -718,13 +804,37 @@ def main() -> None:
804
with open(event_path, encoding="utf-8") as event_file:
805
event = json.load(event_file)
806
721
- if not parse_command(event["comment"]["body"]):
722
- return
723
-
807
github = GitHubClient(
808
os.environ.get("GH_TOKEN", ""),
809
os.environ["GITHUB_REPOSITORY"],
810
)
811
+ event_name = os.environ.get("GITHUB_EVENT_NAME", "issue_comment")
812
+ if event_name == "pull_request_target":
813
+ notify_authorized(
814
+ event,
815
+ github,
816
+ os.environ.get("CI_TRUSTED_USERS", ""),
817
+ )
818
+ return
819
+ if event_name == "workflow_run":
820
+ pull_requests = event["workflow_run"].get("pull_requests") or []
821
+ if not pull_requests:
822
+ return
823
+ pr = github.get_pr(int(pull_requests[0]["number"]))
824
+ notify_authorized(
825
+ {
826
+ "action": "submitted",
827
+ "pull_request": pr,
828
+ "review": {"state": "approved"},
829
+ },
830
+ github,
831
+ os.environ.get("CI_TRUSTED_USERS", ""),
832
+ )
833
+ return
834
+
835
+ if not parse_command(event["comment"]["body"]):
836
+ return
837
+
838
buildkite = BuildkiteClient(
839
os.environ.get("BUILDKITE_API_TOKEN", ""),
840
os.environ.get("BUILDKITE_ORGANIZATION", "vllm"),
.github/workflows/scripts/test_run_ci_command.py
+179
-6
@@ -6,6 +6,7 @@ import unittest
6
from typing import Any
7
8
from run_ci_command import (
9
+ CI_AUTHORIZED_COMMENT_MARKER,
10
COMMAND_RETRY_FAILED,
11
COMMAND_RUN_CI,
12
RETRY_STATES,
@@ -15,6 +16,7 @@ from run_ci_command import (
16
has_trusted_approval,
17
is_active_build,
18
is_build_for_pr,
19
+ notify_authorized,
20
parse_command,
21
parse_trusted_users,
22
run,
@@ -63,8 +65,9 @@ class FakeGitHub:
65
pr: dict[str, Any] | None = None,
66
review_decision: str = "REVIEW_REQUIRED",
67
reviews: list[dict[str, Any]] | None = None,
68
+ comments: list[str] | None = None,
69
) -> None:
67
- self.comments: list[str] = []
70
+ self.comments = comments or []
71
self.permission = permission
72
self.permissions = permissions or {}
73
self.pr = pr or make_pr()
@@ -84,6 +87,15 @@ class FakeGitHub:
87
def list_reviews(self, number: int) -> list[dict[str, Any]]:
88
return self.reviews
89
90
+ def list_issue_comments(self, number: int) -> list[dict[str, Any]]:
91
+ return [
92
+ {
93
+ "body": body,
94
+ "user": {"login": "github-actions[bot]"},
95
+ }
96
+ for body in self.comments
97
+ ]
98
+
99
def list_reactions(self, comment_id: int) -> list[dict[str, Any]]:
100
return []
101
@@ -303,7 +315,7 @@ class RunCiCommandTest(unittest.TestCase):
315
},
316
)
317
306
- def test_ci_run_dispatches_build_with_current_pr_metadata(self) -> None:
318
+ def test_write_reviewer_runs_ci_without_delegation(self) -> None:
319
github = FakeGitHub()
320
buildkite = FakeBuildkite([[], []])
321
run(make_event(COMMAND_RUN_CI), github, buildkite)
@@ -314,6 +326,7 @@ class RunCiCommandTest(unittest.TestCase):
326
"PR #42 /ci run by @reviewer",
327
)
328
self.assertEqual(github.reactions, ["eyes", "rocket"])
329
+ self.assertTrue(github.comments[0].startswith("✅ "))
330
self.assertIn("Buildkite CI #123", github.comments[0])
331
332
def test_unapproved_authors_are_denied_without_buildkite(self) -> None:
@@ -326,10 +339,171 @@ class RunCiCommandTest(unittest.TestCase):
339
run(make_event(COMMAND_RUN_CI, "author"), github, buildkite)
340
341
self.assertEqual(buildkite.list_calls, [])
329
- self.assertEqual(github.reactions, ["eyes", "-1"])
342
+ self.assertEqual(github.reactions, ["eyes"])
343
+ self.assertTrue(github.comments[0].startswith("❌ "))
344
self.assertIn("approve the PR", github.comments[0])
345
332
- def test_ci_retry_uses_latest_current_sha_build(self) -> None:
346
+ run(make_event(COMMAND_RUN_CI, "author"), github, buildkite)
347
+ self.assertEqual(len(github.comments), 1)
348
+
349
+ def test_untrusted_approval_cannot_launch_ci(self) -> None:
350
+ github = FakeGitHub(
351
+ permission="read",
352
+ pr=make_pr(),
353
+ review_decision="APPROVED",
354
+ reviews=[
355
+ {
356
+ "state": "APPROVED",
357
+ "user": {"login": "untrusted-reviewer"},
358
+ }
359
+ ],
360
+ )
361
+ buildkite = FakeBuildkite()
362
+
363
+ run(make_event(COMMAND_RUN_CI, "author"), github, buildkite)
364
+
365
+ self.assertEqual(buildkite.list_calls, [])
366
+ self.assertEqual(github.reactions, ["eyes"])
367
+ self.assertTrue(github.comments[0].startswith("❌ "))
368
+
369
+ def test_ready_label_notifies_author_once(self) -> None:
370
+ pr = make_pr(labels=[{"name": "ready"}])
371
+ event = {
372
+ "action": "labeled",
373
+ "label": {"name": "ready"},
374
+ "pull_request": pr,
375
+ }
376
+ github = FakeGitHub(permission="read", pr=pr)
377
+
378
+ notify_authorized(event, github)
379
+ notify_authorized(event, github)
380
+
381
+ self.assertEqual(len(github.comments), 1)
382
+ self.assertTrue(github.comments[0].startswith("✅ @author"))
383
+ self.assertIn("`/ci run`", github.comments[0])
384
+ self.assertIn("`/ci retry`", github.comments[0])
385
+ self.assertIn(CI_AUTHORIZED_COMMENT_MARKER, github.comments[0])
386
+
387
+ def test_ready_label_does_not_notify_after_trusted_approval(self) -> None:
388
+ pr = make_pr(labels=[{"name": "ready"}])
389
+ event = {
390
+ "action": "labeled",
391
+ "label": {"name": "ready"},
392
+ "pull_request": pr,
393
+ }
394
+ github = FakeGitHub(
395
+ permission="read",
396
+ permissions={"reviewer": "write"},
397
+ pr=pr,
398
+ review_decision="APPROVED",
399
+ reviews=[
400
+ {
401
+ "state": "APPROVED",
402
+ "user": {"login": "reviewer"},
403
+ }
404
+ ],
405
+ )
406
+
407
+ notify_authorized(event, github)
408
+
409
+ self.assertEqual(github.comments, [])
410
+
411
+ def test_trusted_approval_notifies_author_once(self) -> None:
412
+ event = {
413
+ "action": "submitted",
414
+ "pull_request": make_pr(),
415
+ "review": {"state": "approved"},
416
+ }
417
+ github = FakeGitHub(
418
+ permission="read",
419
+ permissions={"reviewer": "write"},
420
+ review_decision="APPROVED",
421
+ reviews=[
422
+ {
423
+ "state": "APPROVED",
424
+ "user": {"login": "reviewer"},
425
+ }
426
+ ],
427
+ )
428
+
429
+ notify_authorized(event, github)
430
+ notify_authorized(event, github)
431
+
432
+ self.assertEqual(len(github.comments), 1)
433
+ self.assertIn("@author", github.comments[0])
434
+
435
+ def test_approval_does_not_notify_when_ready_label_exists(self) -> None:
436
+ pr = make_pr(labels=[{"name": "ready"}])
437
+ event = {
438
+ "action": "submitted",
439
+ "pull_request": pr,
440
+ "review": {"state": "approved"},
441
+ }
442
+ github = FakeGitHub(
443
+ permission="read",
444
+ permissions={"reviewer": "write"},
445
+ pr=pr,
446
+ review_decision="APPROVED",
447
+ reviews=[
448
+ {
449
+ "state": "APPROVED",
450
+ "user": {"login": "reviewer"},
451
+ }
452
+ ],
453
+ )
454
+
455
+ notify_authorized(event, github)
456
+
457
+ self.assertEqual(github.comments, [])
458
+
459
+ def test_untrusted_approval_does_not_notify_author(self) -> None:
460
+ event = {
461
+ "action": "submitted",
462
+ "pull_request": make_pr(),
463
+ "review": {"state": "approved"},
464
+ }
465
+ github = FakeGitHub(
466
+ permission="read",
467
+ review_decision="APPROVED",
468
+ reviews=[
469
+ {
470
+ "state": "APPROVED",
471
+ "user": {"login": "reviewer"},
472
+ }
473
+ ],
474
+ )
475
+
476
+ notify_authorized(event, github)
477
+
478
+ self.assertEqual(github.comments, [])
479
+
480
+ def test_notification_skips_authors_who_already_have_write(self) -> None:
481
+ pr = make_pr(labels=[{"name": "ready"}])
482
+ event = {
483
+ "action": "labeled",
484
+ "label": {"name": "ready"},
485
+ "pull_request": pr,
486
+ }
487
+ github = FakeGitHub(permission="write", pr=pr)
488
+
489
+ notify_authorized(event, github)
490
+
491
+ self.assertEqual(github.comments, [])
492
+
493
+ def test_notification_skips_draft_prs(self) -> None:
494
+ pr = make_pr(draft=True, labels=[{"name": "ready"}])
495
+ event = {
496
+ "action": "labeled",
497
+ "label": {"name": "ready"},
498
+ "pull_request": pr,
499
+ }
500
+ github = FakeGitHub(permission="read", pr=pr)
501
+
502
+ notify_authorized(event, github)
503
+
504
+ self.assertEqual(github.comments, [])
505
+
506
+ def test_ci_retry_retries_failed_jobs_while_build_is_running(self) -> None:
507
github = FakeGitHub(
508
permission="read",
509
pr=make_pr(labels=[{"name": "ready"}]),
@@ -339,10 +513,9 @@ class RunCiCommandTest(unittest.TestCase):
513
[
514
{
515
"created_at": "2026-07-28T01:00:00Z",
342
- "finished_at": "2026-07-28T02:00:00Z",
516
"number": 123,
517
"pull_request": {"id": 42},
345
- "state": "failed",
518
+ "state": "failing",
519
"web_url": "https://buildkite.example/builds/123",
520
}
521
]