[CI] Add run-all comment commands (#51087)
Signed-off-by: khluu <khluu000@gmail.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
Kevin H. Luu committed
Aug 4, 2026 at 17:34 UTC
43c4bdcae9e589fde99984d1d5d03cbdeecf716b
3 files changed
+72
-14
.github/workflows/run-ci-command.yml
+2
@@ -18,6 +18,8 @@ jobs:
18
if: >-
19
github.event.issue.pull_request &&
20
(github.event.comment.body == '/ci run' ||
21
+ github.event.comment.body == '/ci run all' ||
22
+ github.event.comment.body == '/ci run nightly' ||
23
github.event.comment.body == '/ci retry')
24
runs-on: ubuntu-latest
25
steps:
.github/workflows/scripts/run_ci_command.py
+31
-10
@@ -13,7 +13,14 @@ from collections.abc import Callable, Mapping, Sequence
13
from typing import Any
14
15
COMMAND_RUN_CI = "/ci run"
16
+COMMAND_RUN_CI_ALL = "/ci run all"
17
+COMMAND_RUN_CI_NIGHTLY = "/ci run nightly"
18
COMMAND_RETRY_FAILED = "/ci retry"
19
+RUN_CI_COMMAND_ENV = {
20
+ COMMAND_RUN_CI: {},
21
+ COMMAND_RUN_CI_ALL: {"RUN_ALL": "1"},
22
+ COMMAND_RUN_CI_NIGHTLY: {"RUN_ALL": "1", "NIGHTLY": "1"},
23
+}
24
CI_AUTHORIZED_COMMENT_MARKER = "<!-- vllm-ci-authorized -->"
25
READY_LABELS = {"ready", "ready-run-all-tests"}
26
TRUSTED_PERMISSIONS = {"admin", "maintain", "write"}
@@ -383,7 +390,7 @@ class BuildkiteClient:
390
391
392
def parse_command(body: str) -> str | None:
386
- if body in {COMMAND_RUN_CI, COMMAND_RETRY_FAILED}:
393
+ if body in {*RUN_CI_COMMAND_ENV, COMMAND_RETRY_FAILED}:
394
return body
395
return None
396
@@ -486,21 +493,27 @@ def create_build_payload(
493
*,
494
actor: str,
495
comment_id: int,
496
+ command: str = COMMAND_RUN_CI,
497
pr: Mapping[str, Any],
498
) -> dict[str, Any]:
499
+ if command not in RUN_CI_COMMAND_ENV:
500
+ raise ValueError(f"Unsupported run command: {command}")
501
+
502
+ env = {
503
+ "VLLM_CI_GITHUB_COMMENT_ID": str(comment_id),
504
+ "VLLM_CI_TRIGGERED_BY": actor,
505
+ **RUN_CI_COMMAND_ENV[command],
506
+ }
507
return {
508
"commit": pr["head"]["sha"],
509
"branch": pr["head"]["ref"],
494
- "message": f"PR #{pr['number']} {COMMAND_RUN_CI} by @{actor}",
510
+ "message": f"PR #{pr['number']} {command} by @{actor}",
511
"pull_request_id": pr["number"],
512
"pull_request_base_branch": pr["base"]["ref"],
513
"pull_request_repository": pr["head"]["repo"]["clone_url"],
514
"pull_request_labels": [label["name"] for label in pr["labels"]],
515
"ignore_pipeline_branch_filters": True,
500
- "env": {
501
- "VLLM_CI_GITHUB_COMMENT_ID": str(comment_id),
502
- "VLLM_CI_TRIGGERED_BY": actor,
503
- },
516
+ "env": env,
517
"meta_data": {
518
"github-comment-id": str(comment_id),
519
"github-pr-number": str(pr["number"]),
@@ -626,8 +639,12 @@ def notify_authorized(
639
github.add_comment(
640
pr["number"],
641
(
629
- f"✅ @{author}, CI is now available for this PR. Comment `/ci run` "
630
- "to run full CI or `/ci retry` to retry failed jobs.\n\n"
642
+ f"✅ @{author}, CI is now available for this PR.\n\n"
643
+ "- `/ci run` starts a CI build.\n"
644
+ "- `/ci retry` retries failed jobs in the CI build for the current "
645
+ "PR head. If the current head has no CI build, it starts a new CI "
646
+ "build for the current head containing only jobs that failed in "
647
+ "the latest earlier CI build for this PR.\n\n"
648
f"{CI_AUTHORIZED_COMMENT_MARKER}"
649
),
650
)
@@ -638,6 +655,7 @@ def handle_run_ci(
655
actor: str,
656
buildkite: BuildkiteClient,
657
comment_id: int,
658
+ command: str,
659
github: GitHubClient,
660
pr: Mapping[str, Any],
661
) -> str:
@@ -664,13 +682,15 @@ def handle_run_ci(
682
current_pr = github.get_pr(pr["number"])
683
if current_pr["state"] != "open" or current_pr["head"]["sha"] != pr["head"]["sha"]:
684
return (
667
- "The PR head changed while processing the command. Comment `/ci run` again."
685
+ "The PR head changed while processing the command. "
686
+ f"Comment `{command}` again."
687
)
688
689
build = buildkite.create_build(
690
create_build_payload(
691
actor=actor,
692
comment_id=comment_id,
693
+ command=command,
694
pr=current_pr,
695
)
696
)
@@ -835,11 +855,12 @@ def run(
855
return
856
857
print(f"Authorized @{actor}: {reason}")
838
- if command == COMMAND_RUN_CI:
858
+ if command in RUN_CI_COMMAND_ENV:
859
message = handle_run_ci(
860
actor=actor,
861
buildkite=buildkite,
862
comment_id=comment_id,
863
+ command=command,
864
github=github,
865
pr=pr,
866
)
.github/workflows/scripts/test_run_ci_command.py
+39
-4
@@ -12,6 +12,8 @@ from run_ci_command import (
12
CI_AUTHORIZED_COMMENT_MARKER,
13
COMMAND_RETRY_FAILED,
14
COMMAND_RUN_CI,
15
+ COMMAND_RUN_CI_ALL,
16
+ COMMAND_RUN_CI_NIGHTLY,
17
RETRY_STATES,
18
BuildkiteClient,
19
HttpTransport,
@@ -269,11 +271,17 @@ class RunCiCommandTest(unittest.TestCase):
271
272
def test_only_exact_ci_commands_are_accepted(self) -> None:
273
self.assertEqual(parse_command(COMMAND_RUN_CI), COMMAND_RUN_CI)
274
+ self.assertEqual(parse_command(COMMAND_RUN_CI_ALL), COMMAND_RUN_CI_ALL)
275
+ self.assertEqual(
276
+ parse_command(COMMAND_RUN_CI_NIGHTLY),
277
+ COMMAND_RUN_CI_NIGHTLY,
278
+ )
279
self.assertEqual(
280
parse_command(COMMAND_RETRY_FAILED),
281
COMMAND_RETRY_FAILED,
282
)
283
self.assertIsNone(parse_command("/ci run please"))
284
+ self.assertIsNone(parse_command("/ci run all please"))
285
self.assertIsNone(parse_command(" /ci run"))
286
287
def test_write_access_authorizes_reviewers_and_authors(self) -> None:
@@ -435,6 +443,28 @@ class RunCiCommandTest(unittest.TestCase):
443
self.assertTrue(github.comments[0].startswith("✅ "))
444
self.assertIn("Buildkite CI #123", github.comments[0])
445
446
+ def test_run_all_sets_buildkite_environment(self) -> None:
447
+ github = FakeGitHub()
448
+ buildkite = FakeBuildkite([[], []])
449
+
450
+ run(make_event(COMMAND_RUN_CI_ALL), github, buildkite)
451
+
452
+ payload = buildkite.created_builds[0]
453
+ self.assertEqual(payload["message"], "PR #42 /ci run all by @reviewer")
454
+ self.assertEqual(payload["env"]["RUN_ALL"], "1")
455
+ self.assertNotIn("NIGHTLY", payload["env"])
456
+
457
+ def test_run_nightly_sets_buildkite_environment(self) -> None:
458
+ github = FakeGitHub()
459
+ buildkite = FakeBuildkite([[], []])
460
+
461
+ run(make_event(COMMAND_RUN_CI_NIGHTLY), github, buildkite)
462
+
463
+ payload = buildkite.created_builds[0]
464
+ self.assertEqual(payload["message"], "PR #42 /ci run nightly by @reviewer")
465
+ self.assertEqual(payload["env"]["RUN_ALL"], "1")
466
+ self.assertEqual(payload["env"]["NIGHTLY"], "1")
467
+
468
def test_unapproved_authors_are_denied_without_buildkite(self) -> None:
469
github = FakeGitHub(
470
permission="read",
@@ -485,10 +515,15 @@ class RunCiCommandTest(unittest.TestCase):
515
notify_authorized(event, github)
516
517
self.assertEqual(len(github.comments), 1)
488
- self.assertTrue(github.comments[0].startswith("✅ @author"))
489
- self.assertIn("`/ci run`", github.comments[0])
490
- self.assertIn("`/ci retry`", github.comments[0])
491
- self.assertIn(CI_AUTHORIZED_COMMENT_MARKER, github.comments[0])
518
+ comment = github.comments[0]
519
+ self.assertTrue(comment.startswith("✅ @author"))
520
+ self.assertIn("`/ci run` starts a CI build", comment)
521
+ self.assertIn("`/ci retry` retries failed jobs", comment)
522
+ self.assertIn("CI build for the current PR head", comment)
523
+ self.assertIn("only jobs that failed in the latest earlier CI build", comment)
524
+ self.assertNotIn(COMMAND_RUN_CI_ALL, comment)
525
+ self.assertNotIn(COMMAND_RUN_CI_NIGHTLY, comment)
526
+ self.assertIn(CI_AUTHORIZED_COMMENT_MARKER, comment)
527
528
def test_ready_label_does_not_notify_after_trusted_approval(self) -> None:
529
pr = make_pr(labels=[{"name": "ready"}])