[CI] Add /ci cancel command (#51732)
Signed-off-by: khluu <khluu000@gmail.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
Kevin H. Luu committed
Aug 10, 2026 at 14:55 UTC
3c6b2e9c08447a7f839c6aa201d2683ab52d267e
4 files changed
+151
-6
.github/workflows/new_pr_bot.yml
+1
-1
@@ -82,7 +82,7 @@ jobs:
82
'',
83
'PRs do not trigger a full CI run by default. Reviewers with write access and configured trusted contributors can comment `/ci run` whenever CI signals are needed.',
84
'',
85
- 'Once the PR is approved or has the `ready` label, the PR author can also use `/ci run` or `/ci retry`. New commits do not start CI automatically.',
85
+ 'Once the PR is approved or has the `ready` label, the PR author can also use `/ci run`, `/ci retry`, or `/ci cancel`. New commits do not start CI automatically.',
86
'',
87
'If you have any questions, please reach out to us on Slack at https://slack.vllm.ai.',
88
'',
.github/workflows/run-ci-command.yml
+2
-1
@@ -20,7 +20,8 @@ jobs:
20
(github.event.comment.body == '/ci run' ||
21
github.event.comment.body == '/ci run all' ||
22
github.event.comment.body == '/ci run nightly' ||
23
- github.event.comment.body == '/ci retry')
23
+ github.event.comment.body == '/ci retry' ||
24
+ github.event.comment.body == '/ci cancel')
25
runs-on: [self-hosted, linux, x64, vllm-runners]
26
steps:
27
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
.github/workflows/scripts/run_ci_command.py
+54
-4
@@ -16,6 +16,7 @@ COMMAND_RUN_CI = "/ci run"
16
COMMAND_RUN_CI_ALL = "/ci run all"
17
COMMAND_RUN_CI_NIGHTLY = "/ci run nightly"
18
COMMAND_RETRY_FAILED = "/ci retry"
19
+COMMAND_CANCEL_CI = "/ci cancel"
20
RUN_CI_COMMAND_ENV = {
21
COMMAND_RUN_CI: {},
22
COMMAND_RUN_CI_ALL: {"RUN_ALL": "1"},
@@ -35,6 +36,7 @@ ACTIVE_BUILD_STATES = {
36
"waiting_failed",
37
}
38
RETRY_STATES = "failed,timed_out,expired"
39
+CANCELABLE_BUILD_STATES = ("scheduled", "running", "failing")
40
SETUP_STEP_KEYS = {
41
"ensure-ci-base-amd",
42
"pre-commit",
@@ -335,7 +337,9 @@ class BuildkiteClient:
337
self,
338
commit: str | None,
339
*,
340
+ branch: str | None = None,
341
metadata: tuple[str, str] | None = None,
342
+ states: Sequence[str] = (),
343
) -> list[dict[str, Any]]:
344
query = [
345
("exclude_jobs", "true"),
@@ -344,9 +348,12 @@ class BuildkiteClient:
348
]
349
if commit:
350
query.append(("commit", commit))
351
+ if branch:
352
+ query.append(("branch", branch))
353
if metadata:
354
key, value = metadata
355
query.append((f"meta_data[{key}]", value))
356
+ query.extend(("state[]", state) for state in states)
357
response = self._request(query=query)
358
if not isinstance(response, list):
359
raise ApiError(None, "Buildkite API returned an invalid build list.")
@@ -367,6 +374,10 @@ class BuildkiteClient:
374
path=f"/{number}/retry_failed_jobs",
375
)
376
377
+ def cancel_build(self, build_number: int) -> dict[str, Any]:
378
+ number = urllib.parse.quote(str(build_number), safe="")
379
+ return self._request(method="PUT", path=f"/{number}/cancel")
380
+
381
def list_failed_jobs(self, build_number: int) -> list[dict[str, Any]]:
382
number = urllib.parse.quote(str(build_number), safe="")
383
query = [
@@ -397,7 +408,7 @@ class BuildkiteClient:
408
409
410
def parse_command(body: str) -> str | None:
400
- if body in {*RUN_CI_COMMAND_ENV, COMMAND_RETRY_FAILED}:
411
+ if body in {*RUN_CI_COMMAND_ENV, COMMAND_RETRY_FAILED, COMMAND_CANCEL_CI}:
412
return body
413
return None
414
@@ -432,7 +443,7 @@ def authorize(
443
if actor.casefold() != pr["user"]["login"].casefold():
444
return (
445
False,
435
- "Only reviewers with write access can run CI before it is "
446
+ "Only reviewers with write access can use CI commands before CI is "
447
"delegated to the PR author.",
448
)
449
if pr["draft"]:
@@ -650,7 +661,9 @@ def notify_authorized(
661
"- `/ci retry` retries failed jobs in the CI build for the current "
662
"PR head. If the current head has no CI build, it starts a new CI "
663
"build for the current head containing only jobs that failed in "
653
- "the latest earlier CI build for this PR.\n\n"
664
+ "the latest earlier CI build for this PR.\n"
665
+ "- `/ci cancel` cancels scheduled or running CI builds for this PR "
666
+ "branch.\n\n"
667
f"{CI_AUTHORIZED_COMMENT_MARKER}"
668
),
669
)
@@ -837,6 +850,38 @@ def handle_retry_failed(
850
)
851
852
853
+def handle_cancel_ci(
854
+ *,
855
+ buildkite: BuildkiteClient,
856
+ pr: Mapping[str, Any],
857
+) -> str:
858
+ branch = pr["head"]["ref"]
859
+ builds = buildkite.list_builds(
860
+ None,
861
+ branch=branch,
862
+ states=CANCELABLE_BUILD_STATES,
863
+ )
864
+ cancelable_builds = [
865
+ build
866
+ for build in builds
867
+ if build.get("branch") == branch
868
+ and is_build_for_pr(build, pr["number"])
869
+ and build.get("state") in CANCELABLE_BUILD_STATES
870
+ ]
871
+ if not cancelable_builds:
872
+ return f"No cancelable CI build is running for branch `{branch}`."
873
+
874
+ for build in cancelable_builds:
875
+ buildkite.cancel_build(build["number"])
876
+
877
+ links = ", ".join(
878
+ f"[#{build['number']}]({build['web_url']})" for build in cancelable_builds
879
+ )
880
+ count = len(cancelable_builds)
881
+ noun = "build" if count == 1 else "builds"
882
+ return f"Requested cancellation of {count} CI {noun} for `{branch}`: {links}."
883
+
884
+
885
def run(
886
event: Mapping[str, Any],
887
github: GitHubClient,
@@ -904,7 +949,7 @@ def run(
949
github=github,
950
pr=pr,
951
)
907
- else:
952
+ elif command == COMMAND_RETRY_FAILED:
953
message = handle_retry_failed(
954
actor=actor,
955
buildkite=buildkite,
@@ -912,6 +957,11 @@ def run(
957
github=github,
958
pr=pr,
959
)
960
+ else:
961
+ message = handle_cancel_ci(
962
+ buildkite=buildkite,
963
+ pr=pr,
964
+ )
965
add_reaction_safely(github, comment_id, "rocket")
966
github.add_comment(issue_number, f"✅ {message}")
967
except Exception:
.github/workflows/scripts/test_run_ci_command.py
+94
@@ -9,7 +9,9 @@ from typing import Any
9
from unittest.mock import patch
10
11
from run_ci_command import (
12
+ CANCELABLE_BUILD_STATES,
13
CI_AUTHORIZED_COMMENT_MARKER,
14
+ COMMAND_CANCEL_CI,
15
COMMAND_RETRY_FAILED,
16
COMMAND_RUN_CI,
17
COMMAND_RUN_CI_ALL,
@@ -136,12 +138,15 @@ class FakeBuildkite:
138
self.job_list_calls: list[int] = []
139
self.list_calls: list[tuple[str | None, tuple[str, str] | None]] = []
140
self.retry_calls: list[tuple[int, str]] = []
141
+ self.cancel_calls: list[int] = []
142
143
def list_builds(
144
self,
145
commit: str | None,
146
*,
147
+ branch: str | None = None,
148
metadata: tuple[str, str] | None = None,
149
+ states: tuple[str, ...] = (),
150
) -> list[dict[str, Any]]:
151
self.list_calls.append((commit, metadata))
152
return self.build_lists.pop(0)
@@ -161,6 +166,10 @@ class FakeBuildkite:
166
self.retry_calls.append((build_number, states))
167
return {"retried_jobs_count": 3}
168
169
+ def cancel_build(self, build_number: int) -> dict[str, Any]:
170
+ self.cancel_calls.append(build_number)
171
+ return {"number": build_number, "state": "canceling"}
172
+
173
def list_failed_jobs(self, build_number: int) -> list[dict[str, Any]]:
174
self.job_list_calls.append(build_number)
175
return self.failed_job_lists.pop(0)
@@ -292,8 +301,10 @@ class RunCiCommandTest(unittest.TestCase):
301
parse_command(COMMAND_RETRY_FAILED),
302
COMMAND_RETRY_FAILED,
303
)
304
+ self.assertEqual(parse_command(COMMAND_CANCEL_CI), COMMAND_CANCEL_CI)
305
self.assertIsNone(parse_command("/ci run please"))
306
self.assertIsNone(parse_command("/ci run all please"))
307
+ self.assertIsNone(parse_command("/ci cancel please"))
308
self.assertIsNone(parse_command(" /ci run"))
309
310
def test_write_access_authorizes_reviewers_and_authors(self) -> None:
@@ -531,6 +542,7 @@ class RunCiCommandTest(unittest.TestCase):
542
self.assertTrue(comment.startswith("✅ @author"))
543
self.assertIn("`/ci run` starts a CI build", comment)
544
self.assertIn("`/ci retry` retries failed jobs", comment)
545
+ self.assertIn("`/ci cancel` cancels scheduled or running", comment)
546
self.assertIn("CI build for the current PR head", comment)
547
self.assertIn("only jobs that failed in the latest earlier CI build", comment)
548
self.assertNotIn(COMMAND_RUN_CI_ALL, comment)
@@ -854,6 +866,82 @@ class RunCiCommandTest(unittest.TestCase):
866
self.assertTrue(call["url"].endswith("/123/retry_failed_jobs"))
867
self.assertEqual(call["body"], {"states": RETRY_STATES})
868
869
+ def test_ci_cancel_cancels_active_builds_for_pr_branch(self) -> None:
870
+ github = FakeGitHub()
871
+ buildkite = FakeBuildkite(
872
+ [
873
+ [
874
+ {
875
+ "branch": "feature",
876
+ "number": 123,
877
+ "pull_request": {"id": 42},
878
+ "state": "running",
879
+ "web_url": "https://buildkite.example/builds/123",
880
+ },
881
+ {
882
+ "branch": "feature",
883
+ "number": 124,
884
+ "meta_data": {"github-pr-number": "42"},
885
+ "state": "failing",
886
+ "web_url": "https://buildkite.example/builds/124",
887
+ },
888
+ {
889
+ "branch": "feature",
890
+ "number": 125,
891
+ "pull_request": {"id": 43},
892
+ "state": "running",
893
+ "web_url": "https://buildkite.example/builds/125",
894
+ },
895
+ {
896
+ "branch": "other-branch",
897
+ "number": 126,
898
+ "pull_request": {"id": 42},
899
+ "state": "running",
900
+ "web_url": "https://buildkite.example/builds/126",
901
+ },
902
+ {
903
+ "branch": "feature",
904
+ "number": 127,
905
+ "pull_request": {"id": 42},
906
+ "state": "passed",
907
+ "web_url": "https://buildkite.example/builds/127",
908
+ },
909
+ ]
910
+ ]
911
+ )
912
+
913
+ run(make_event(COMMAND_CANCEL_CI), github, buildkite)
914
+
915
+ self.assertEqual(buildkite.cancel_calls, [123, 124])
916
+ self.assertIn("Requested cancellation of 2 CI builds", github.comments[0])
917
+ self.assertIn("#123", github.comments[0])
918
+ self.assertIn("#124", github.comments[0])
919
+
920
+ def test_ci_cancel_is_a_noop_without_active_builds(self) -> None:
921
+ github = FakeGitHub()
922
+ buildkite = FakeBuildkite([[]])
923
+
924
+ run(make_event(COMMAND_CANCEL_CI), github, buildkite)
925
+
926
+ self.assertEqual(buildkite.cancel_calls, [])
927
+ self.assertIn("No cancelable CI build is running", github.comments[0])
928
+
929
+ def test_buildkite_cancel_uses_cancel_build_endpoint(self) -> None:
930
+ transport = FakeTransport({"number": 123, "state": "canceling"})
931
+ client = BuildkiteClient(
932
+ "secret",
933
+ "vllm",
934
+ "ci",
935
+ transport=transport,
936
+ )
937
+
938
+ client.cancel_build(123)
939
+
940
+ call = transport.calls[0]
941
+ self.assertEqual(call["method"], "PUT")
942
+ self.assertTrue(call["url"].endswith("/123/cancel"))
943
+ self.assertIsNone(call["body"])
944
+
945
def test_buildkite_list_builds_allows_query_on_builds_endpoint(self) -> None:
946
transport = FakeTransport([])
947
client = BuildkiteClient(
@@ -865,14 +953,20 @@ class RunCiCommandTest(unittest.TestCase):
953
954
builds = client.list_builds(
955
"current-commit",
956
+ branch="feature",
957
metadata=("github-pr-number", "42"),
958
+ states=CANCELABLE_BUILD_STATES,
959
)
960
961
self.assertEqual(builds, [])
962
url = transport.calls[0]["url"]
963
self.assertIn("?exclude_jobs=true", url)
964
self.assertIn("commit=current-commit", url)
965
+ self.assertIn("branch=feature", url)
966
self.assertIn("meta_data%5Bgithub-pr-number%5D=42", url)
967
+ self.assertIn("state%5B%5D=scheduled", url)
968
+ self.assertIn("state%5B%5D=running", url)
969
+ self.assertIn("state%5B%5D=failing", url)
970
971
def test_buildkite_failed_jobs_follow_cursor_pagination(self) -> None:
972
next_url = (