@tduncan / Nvidia-Optimization-Stack-VLLM / commits / 7f4c52f2ba

[CI] Add comment-based Buildkite triggers (#50132)

Signed-off-by: khluu <khluu000@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com>

Kevin H. Luu committed Jul 28, 2026 at 18:41 UTC 7f4c52f2ba38a77d5341fb90a31c2cedfc0066b8
6 files changed +989 -5
.github/workflows/new_pr_bot.yml
+2 -2
@@ -80,9 +80,9 @@ jobs:
80 '',
81 '\u{1f4ac} Join our developer Slack at https://slack.vllm.ai to discuss your PR in `#pr-reviews`, coordinate on features in `#feat-` channels, or join special interest groups in `#sig-` channels.',
82 '',
83 - 'PRs do not trigger a full CI run by default. Once the PR is approved and ready to go, your PR reviewer(s) can run CI to test the changes comprehensively before merging.',
83 + 'PRs do not trigger a full CI run by default. Reviewers with write access and configured trusted contributors can comment `/ci run` whenever CI signals are needed.',
84 '',
85 - 'To run CI, PR reviewers can either: Add `ready` label to the PR or enable auto-merge.',
85 + 'Once the PR is approved or has the `ready` label, the PR author can also use `/ci run` or `/ci retry`. New commits do not start CI automatically.',
86 '',
87 'If you have any questions, please reach out to us on Slack at https://slack.vllm.ai.',
88 '',
.github/workflows/pre-commit.yml
+1 -1
@@ -41,7 +41,7 @@ jobs:
41 if (hasReadyLabel || hasVerifiedLabel || mergedCount >= 4) {
42 core.info(`Check passed: verified label=${hasVerifiedLabel}, ready label=${hasReadyLabel}, 4+ merged PRs=${mergedCount >= 4}`);
43 } else {
44 - core.setFailed(`PR must have the 'verified', 'ready', or 'ready-run-all-tests' label (the ready labels also trigger tests) or the author must have at least 4 merged PRs (found ${mergedCount}).`);
44 + core.setFailed(`PR must have the 'verified', 'ready', or 'ready-run-all-tests' label to run pre-commit, or the author must have at least 4 merged PRs (found ${mergedCount}).`);
45 }
46
47 pre-commit:
.github/workflows/run-ci-command.yml new
+40
@@ -0,0 +1,40 @@
1 +name: Run CI from PR comment
2 +
3 +on:
4 + issue_comment:
5 + types: [created]
6 +
7 +concurrency:
8 + group: run-ci-comment-${{ github.event.issue.number }}
9 + cancel-in-progress: false
10 +
11 +permissions:
12 + contents: read
13 + issues: write
14 + pull-requests: read
15 +
16 +jobs:
17 + run-ci-command:
18 + if: >-
19 + github.event.issue.pull_request &&
20 + (github.event.comment.body == '/ci run' ||
21 + github.event.comment.body == '/ci retry')
22 + runs-on: ubuntu-latest
23 + steps:
24 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
25 + with:
26 + ref: ${{ github.event.repository.default_branch }}
27 + persist-credentials: false
28 + - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
29 + with:
30 + python-version: "3.12"
31 + - name: Authorize and run CI command
32 + run: >-
33 + uv run --no-project --python 3.12
34 + .github/workflows/scripts/run_ci_command.py
35 + env:
36 + BUILDKITE_API_TOKEN: ${{ secrets.BUILDKITE_API_TOKEN }}
37 + BUILDKITE_ORGANIZATION: vllm
38 + BUILDKITE_PIPELINE: ci
39 + CI_TRUSTED_USERS: ${{ vars.CI_TRUSTED_USERS }}
40 + GH_TOKEN: ${{ github.token }}
.github/workflows/scripts/run_ci_command.py new
+580
@@ -0,0 +1,580 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +import json
5 +import os
6 +import sys
7 +import urllib.error
8 +import urllib.parse
9 +import urllib.request
10 +from collections.abc import Mapping, Sequence
11 +from typing import Any
12 +
13 +COMMAND_RUN_CI = "/ci run"
14 +COMMAND_RETRY_FAILED = "/ci retry"
15 +READY_LABELS = {"ready", "ready-run-all-tests"}
16 +TRUSTED_PERMISSIONS = {"admin", "maintain", "write"}
17 +ACTIVE_BUILD_STATES = {
18 + "blocked",
19 + "creating",
20 + "scheduled",
21 + "running",
22 + "failing",
23 + "canceling",
24 + "waiting",
25 + "waiting_failed",
26 +}
27 +RETRY_STATES = "failed,timed_out,expired"
28 +
29 +
30 +class ApiError(RuntimeError):
31 + def __init__(self, status: int | None, message: str) -> None:
32 + super().__init__(message)
33 + self.status = status
34 +
35 +
36 +class HttpTransport:
37 + def request(
38 + self,
39 + url: str,
40 + *,
41 + body: Mapping[str, Any] | None = None,
42 + headers: Mapping[str, str] | None = None,
43 + method: str = "GET",
44 + ) -> Any:
45 + data = None if body is None else json.dumps(body).encode()
46 + request = urllib.request.Request(
47 + url,
48 + data=data,
49 + headers=dict(headers or {}),
50 + method=method,
51 + )
52 + try:
53 + with urllib.request.urlopen(request, timeout=30) as response:
54 + response_body = response.read().decode()
55 + except urllib.error.HTTPError as error:
56 + response_body = error.read().decode()
57 + message = self._error_message(response_body, error.reason)
58 + raise ApiError(
59 + error.code,
60 + f"API returned {error.code}: {message}",
61 + ) from error
62 + except urllib.error.URLError as error:
63 + raise ApiError(None, f"API request failed: {error.reason}") from error
64 +
65 + if not response_body:
66 + return None
67 + try:
68 + return json.loads(response_body)
69 + except json.JSONDecodeError as error:
70 + raise ApiError(None, "API returned a non-JSON response.") from error
71 +
72 + @staticmethod
73 + def _error_message(response_body: str, fallback: str) -> str:
74 + try:
75 + parsed = json.loads(response_body)
76 + except json.JSONDecodeError:
77 + return fallback
78 + return str(parsed.get("message", fallback))
79 +
80 +
81 +class GitHubClient:
82 + def __init__(
83 + self,
84 + token: str,
85 + repository: str,
86 + transport: HttpTransport | None = None,
87 + ) -> None:
88 + if not token:
89 + raise RuntimeError("GH_TOKEN is not set.")
90 + self.owner, self.repo = repository.split("/", maxsplit=1)
91 + self.transport = transport or HttpTransport()
92 + self.headers = {
93 + "Accept": "application/vnd.github+json",
94 + "Authorization": f"Bearer {token}",
95 + "Content-Type": "application/json",
96 + "User-Agent": "vllm-ci-command",
97 + "X-GitHub-Api-Version": "2022-11-28",
98 + }
99 +
100 + def _request(
101 + self,
102 + path: str,
103 + *,
104 + body: Mapping[str, Any] | None = None,
105 + method: str = "GET",
106 + ) -> Any:
107 + return self.transport.request(
108 + f"https://api.github.com{path}",
109 + body=body,
110 + headers=self.headers,
111 + method=method,
112 + )
113 +
114 + def _repo_path(self, suffix: str) -> str:
115 + owner = urllib.parse.quote(self.owner, safe="")
116 + repo = urllib.parse.quote(self.repo, safe="")
117 + return f"/repos/{owner}/{repo}{suffix}"
118 +
119 + def _paginate(self, path: str) -> list[dict[str, Any]]:
120 + results: list[dict[str, Any]] = []
121 + separator = "&" if "?" in path else "?"
122 + for page in range(1, 101):
123 + response = self._request(f"{path}{separator}per_page=100&page={page}")
124 + if not isinstance(response, list):
125 + raise ApiError(None, "GitHub API returned an invalid list response.")
126 + results.extend(response)
127 + if len(response) < 100:
128 + return results
129 + raise ApiError(None, "GitHub API pagination exceeded 10,000 results.")
130 +
131 + def get_pr(self, number: int) -> dict[str, Any]:
132 + return self._request(self._repo_path(f"/pulls/{number}"))
133 +
134 + def get_permission(self, actor: str) -> str:
135 + username = urllib.parse.quote(actor, safe="")
136 + try:
137 + response = self._request(
138 + self._repo_path(f"/collaborators/{username}/permission")
139 + )
140 + except ApiError as error:
141 + if error.status == 404:
142 + return "none"
143 + raise
144 + return str(response["permission"])
145 +
146 + def get_review_decision(self, number: int) -> str | None:
147 + query = """
148 + query($owner: String!, $repo: String!, $number: Int!) {
149 + repository(owner: $owner, name: $repo) {
150 + pullRequest(number: $number) {
151 + reviewDecision
152 + }
153 + }
154 + }
155 + """
156 + response = self._request(
157 + "/graphql",
158 + body={
159 + "query": query,
160 + "variables": {
161 + "number": number,
162 + "owner": self.owner,
163 + "repo": self.repo,
164 + },
165 + },
166 + method="POST",
167 + )
168 + return response["data"]["repository"]["pullRequest"]["reviewDecision"]
169 +
170 + def list_reviews(self, number: int) -> list[dict[str, Any]]:
171 + return self._paginate(self._repo_path(f"/pulls/{number}/reviews"))
172 +
173 + def list_reactions(self, comment_id: int) -> list[dict[str, Any]]:
174 + return self._paginate(
175 + self._repo_path(f"/issues/comments/{comment_id}/reactions")
176 + )
177 +
178 + def add_reaction(self, comment_id: int, content: str) -> None:
179 + self._request(
180 + self._repo_path(f"/issues/comments/{comment_id}/reactions"),
181 + body={"content": content},
182 + method="POST",
183 + )
184 +
185 + def add_comment(self, issue_number: int, body: str) -> None:
186 + self._request(
187 + self._repo_path(f"/issues/{issue_number}/comments"),
188 + body={"body": body},
189 + method="POST",
190 + )
191 +
192 +
193 +class BuildkiteClient:
194 + def __init__(
195 + self,
196 + token: str,
197 + organization: str,
198 + pipeline: str,
199 + transport: HttpTransport | None = None,
200 + ) -> None:
201 + self.token = token
202 + self.transport = transport or HttpTransport()
203 + organization = urllib.parse.quote(organization, safe="")
204 + pipeline = urllib.parse.quote(pipeline, safe="")
205 + self.base_url = (
206 + "https://api.buildkite.com/v2/organizations/"
207 + f"{organization}/pipelines/{pipeline}/builds"
208 + )
209 +
210 + def _request(
211 + self,
212 + *,
213 + body: Mapping[str, Any] | None = None,
214 + method: str = "GET",
215 + path: str = "",
216 + query: Sequence[tuple[str, str]] = (),
217 + ) -> Any:
218 + if not self.token:
219 + raise RuntimeError("The BUILDKITE_API_TOKEN repository secret is not set.")
220 + url = f"{self.base_url}{path}"
221 + if query:
222 + url = f"{url}?{urllib.parse.urlencode(query)}"
223 + return self.transport.request(
224 + url,
225 + body=body,
226 + headers={
227 + "Authorization": f"Bearer {self.token}",
228 + "Content-Type": "application/json",
229 + "User-Agent": "vllm-ci-command",
230 + },
231 + method=method,
232 + )
233 +
234 + def list_builds(
235 + self,
236 + commit: str,
237 + *,
238 + metadata: tuple[str, str] | None = None,
239 + ) -> list[dict[str, Any]]:
240 + query = [
241 + ("commit", commit),
242 + ("exclude_jobs", "true"),
243 + ("exclude_pipeline", "true"),
244 + ("per_page", "100"),
245 + ]
246 + if metadata:
247 + key, value = metadata
248 + query.append((f"meta_data[{key}]", value))
249 + response = self._request(query=query)
250 + if not isinstance(response, list):
251 + raise ApiError(None, "Buildkite API returned an invalid build list.")
252 + return response
253 +
254 + def create_build(self, body: Mapping[str, Any]) -> dict[str, Any]:
255 + return self._request(body=body, method="POST")
256 +
257 + def retry_failed_jobs(
258 + self,
259 + build_number: int,
260 + states: str,
261 + ) -> dict[str, Any]:
262 + number = urllib.parse.quote(str(build_number), safe="")
263 + return self._request(
264 + body={"states": states},
265 + method="PUT",
266 + path=f"/{number}/retry_failed_jobs",
267 + )
268 +
269 +
270 +def parse_command(body: str) -> str | None:
271 + if body in {COMMAND_RUN_CI, COMMAND_RETRY_FAILED}:
272 + return body
273 + return None
274 +
275 +
276 +def parse_trusted_users(value: str = "") -> set[str]:
277 + return {
278 + user.casefold() for item in value.split(",") for user in item.split() if user
279 + }
280 +
281 +
282 +def has_ready_label(pr: Mapping[str, Any]) -> bool:
283 + return any(label["name"] in READY_LABELS for label in pr["labels"])
284 +
285 +
286 +def is_trusted_permission(permission: str) -> bool:
287 + return permission in TRUSTED_PERMISSIONS
288 +
289 +
290 +def authorize(
291 + *,
292 + actor: str,
293 + permission: str,
294 + pr: Mapping[str, Any],
295 + trusted_approval: bool = False,
296 + trusted_users: set[str] | None = None,
297 +) -> tuple[bool, str]:
298 + trusted_users = trusted_users or set()
299 + if is_trusted_permission(permission):
300 + return True, f"repository {permission} permission"
301 + if actor.casefold() in trusted_users:
302 + return True, "configured trusted contributor"
303 + if actor.casefold() != pr["user"]["login"].casefold():
304 + return (
305 + False,
306 + "Only reviewers with write access can run CI before it is "
307 + "delegated to the PR author.",
308 + )
309 + if pr["draft"]:
310 + return False, "PR authors cannot run CI while the PR is a draft."
311 + if has_ready_label(pr):
312 + return True, "ready label"
313 + if trusted_approval:
314 + return True, "approval from a trusted reviewer"
315 + return (
316 + False,
317 + "A reviewer with write access must run `/ci run`, approve the PR, "
318 + "or add the `ready` label first.",
319 + )
320 +
321 +
322 +def has_trusted_approval(
323 + github: GitHubClient,
324 + number: int,
325 + trusted_users: set[str],
326 +) -> bool:
327 + if github.get_review_decision(number) != "APPROVED":
328 + return False
329 +
330 + latest_review_states: dict[str, tuple[str, str]] = {}
331 + for review in github.list_reviews(number):
332 + user = review.get("user") or {}
333 + login = user.get("login")
334 + state = review.get("state")
335 + if login and state in {"APPROVED", "CHANGES_REQUESTED", "DISMISSED"}:
336 + latest_review_states[login.casefold()] = (login, state)
337 +
338 + for login, state in latest_review_states.values():
339 + if state != "APPROVED":
340 + continue
341 + if login.casefold() in trusted_users:
342 + return True
343 + if is_trusted_permission(github.get_permission(login)):
344 + return True
345 + return False
346 +
347 +
348 +def is_build_for_pr(build: Mapping[str, Any], pr_number: int) -> bool:
349 + pull_request = build.get("pull_request")
350 + if isinstance(pull_request, Mapping):
351 + build_pr_number = pull_request.get("id", pull_request.get("number"))
352 + if build_pr_number is not None:
353 + return str(build_pr_number) == str(pr_number)
354 + metadata = build.get("meta_data") or {}
355 + return str(metadata.get("github-pr-number")) == str(pr_number)
356 +
357 +
358 +def is_active_build(build: Mapping[str, Any]) -> bool:
359 + return bool(build.get("blocked")) or build.get("state") in ACTIVE_BUILD_STATES
360 +
361 +
362 +def select_latest_build(
363 + builds: Sequence[dict[str, Any]],
364 + pr_number: int,
365 +) -> dict[str, Any] | None:
366 + matching = [build for build in builds if is_build_for_pr(build, pr_number)]
367 + return max(matching, key=lambda build: build.get("created_at", ""), default=None)
368 +
369 +
370 +def create_build_payload(
371 + *,
372 + actor: str,
373 + comment_id: int,
374 + pr: Mapping[str, Any],
375 +) -> dict[str, Any]:
376 + return {
377 + "commit": pr["head"]["sha"],
378 + "branch": pr["head"]["ref"],
379 + "message": f"PR #{pr['number']} {COMMAND_RUN_CI} by @{actor}",
380 + "pull_request_id": pr["number"],
381 + "pull_request_base_branch": pr["base"]["ref"],
382 + "pull_request_repository": pr["head"]["repo"]["clone_url"],
383 + "pull_request_labels": [label["name"] for label in pr["labels"]],
384 + "env": {
385 + "VLLM_CI_GITHUB_COMMENT_ID": str(comment_id),
386 + "VLLM_CI_TRIGGERED_BY": actor,
387 + },
388 + "meta_data": {
389 + "github-comment-id": str(comment_id),
390 + "github-pr-number": str(pr["number"]),
391 + "github-triggered-by": actor,
392 + },
393 + }
394 +
395 +
396 +def add_reaction_safely(
397 + github: GitHubClient,
398 + comment_id: int,
399 + content: str,
400 +) -> None:
401 + try:
402 + github.add_reaction(comment_id, content)
403 + except Exception as error:
404 + print(f"Could not add {content} reaction: {error}", file=sys.stderr)
405 +
406 +
407 +def is_already_handled(github: GitHubClient, comment_id: int) -> bool:
408 + return any(
409 + reaction.get("content") in {"rocket", "-1"}
410 + and (reaction.get("user") or {}).get("login") == "github-actions[bot]"
411 + for reaction in github.list_reactions(comment_id)
412 + )
413 +
414 +
415 +def handle_run_ci(
416 + *,
417 + actor: str,
418 + buildkite: BuildkiteClient,
419 + comment_id: int,
420 + github: GitHubClient,
421 + pr: Mapping[str, Any],
422 +) -> str:
423 + duplicate_builds = buildkite.list_builds(
424 + pr["head"]["sha"],
425 + metadata=("github-comment-id", str(comment_id)),
426 + )
427 + duplicate = select_latest_build(duplicate_builds, pr["number"])
428 + if duplicate:
429 + return f"CI was already requested by this comment: {duplicate['web_url']}"
430 +
431 + current_builds = buildkite.list_builds(pr["head"]["sha"])
432 + active_build = next(
433 + (
434 + build
435 + for build in current_builds
436 + if is_build_for_pr(build, pr["number"]) and is_active_build(build)
437 + ),
438 + None,
439 + )
440 + if active_build:
441 + return f"CI is already running for this commit: {active_build['web_url']}"
442 +
443 + current_pr = github.get_pr(pr["number"])
444 + if current_pr["state"] != "open" or current_pr["head"]["sha"] != pr["head"]["sha"]:
445 + return (
446 + "The PR head changed while processing the command. Comment `/ci run` again."
447 + )
448 +
449 + build = buildkite.create_build(
450 + create_build_payload(
451 + actor=actor,
452 + comment_id=comment_id,
453 + pr=current_pr,
454 + )
455 + )
456 + return (
457 + f"Triggered [Buildkite CI #{build['number']}]({build['web_url']}) "
458 + f"for commit `{current_pr['head']['sha'][:12]}`."
459 + )
460 +
461 +
462 +def handle_retry_failed(
463 + *,
464 + buildkite: BuildkiteClient,
465 + pr: Mapping[str, Any],
466 +) -> str:
467 + builds = buildkite.list_builds(pr["head"]["sha"])
468 + build = select_latest_build(builds, pr["number"])
469 + if not build:
470 + return "No CI build exists for the current PR commit. Use `/ci run` first."
471 + if not build.get("finished_at") or is_active_build(build):
472 + return f"CI is still running for this commit: {build['web_url']}"
473 +
474 + retried = buildkite.retry_failed_jobs(build["number"], RETRY_STATES)
475 + if retried["retried_jobs_count"] == 0:
476 + return (
477 + f"No failed, timed-out, or expired jobs need retrying: {build['web_url']}"
478 + )
479 + return (
480 + f"Queued {retried['retried_jobs_count']} failed job(s) for retry in "
481 + f"[Buildkite CI #{build['number']}]({build['web_url']})."
482 + )
483 +
484 +
485 +def run(
486 + event: Mapping[str, Any],
487 + github: GitHubClient,
488 + buildkite: BuildkiteClient,
489 + trusted_users_value: str = "",
490 +) -> None:
491 + command = parse_command(event["comment"]["body"])
492 + if not command or "pull_request" not in event["issue"]:
493 + return
494 +
495 + issue_number = event["issue"]["number"]
496 + comment_id = event["comment"]["id"]
497 + actor = event["comment"]["user"]["login"]
498 +
499 + if is_already_handled(github, comment_id):
500 + print(f"Comment {comment_id} was already handled.")
501 + return
502 + add_reaction_safely(github, comment_id, "eyes")
503 +
504 + try:
505 + pr = github.get_pr(issue_number)
506 + permission = github.get_permission(actor)
507 + if pr["state"] != "open":
508 + github.add_comment(issue_number, "CI commands require an open PR.")
509 + return
510 +
511 + trusted_users = parse_trusted_users(trusted_users_value)
512 + should_check_approval = (
513 + not is_trusted_permission(permission)
514 + and actor.casefold() not in trusted_users
515 + and actor.casefold() == pr["user"]["login"].casefold()
516 + and not pr["draft"]
517 + and not has_ready_label(pr)
518 + )
519 + trusted_approval = should_check_approval and has_trusted_approval(
520 + github,
521 + issue_number,
522 + trusted_users,
523 + )
524 + allowed, reason = authorize(
525 + actor=actor,
526 + permission=permission,
527 + pr=pr,
528 + trusted_approval=trusted_approval,
529 + trusted_users=trusted_users,
530 + )
531 + if not allowed:
532 + add_reaction_safely(github, comment_id, "-1")
533 + github.add_comment(issue_number, f"@{actor}, {reason}")
534 + return
535 +
536 + print(f"Authorized @{actor}: {reason}")
537 + if command == COMMAND_RUN_CI:
538 + message = handle_run_ci(
539 + actor=actor,
540 + buildkite=buildkite,
541 + comment_id=comment_id,
542 + github=github,
543 + pr=pr,
544 + )
545 + else:
546 + message = handle_retry_failed(buildkite=buildkite, pr=pr)
547 + add_reaction_safely(github, comment_id, "rocket")
548 + github.add_comment(issue_number, message)
549 + except Exception:
550 + add_reaction_safely(github, comment_id, "confused")
551 + raise
552 +
553 +
554 +def main() -> None:
555 + event_path = os.environ["GITHUB_EVENT_PATH"]
556 + with open(event_path, encoding="utf-8") as event_file:
557 + event = json.load(event_file)
558 +
559 + if not parse_command(event["comment"]["body"]):
560 + return
561 +
562 + github = GitHubClient(
563 + os.environ.get("GH_TOKEN", ""),
564 + os.environ["GITHUB_REPOSITORY"],
565 + )
566 + buildkite = BuildkiteClient(
567 + os.environ.get("BUILDKITE_API_TOKEN", ""),
568 + os.environ.get("BUILDKITE_ORGANIZATION", "vllm"),
569 + os.environ.get("BUILDKITE_PIPELINE", "ci"),
570 + )
571 + run(
572 + event,
573 + github,
574 + buildkite,
575 + os.environ.get("CI_TRUSTED_USERS", ""),
576 + )
577 +
578 +
579 +if __name__ == "__main__":
580 + main()
.github/workflows/scripts/test_run_ci_command.py new
+362
@@ -0,0 +1,362 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +import unittest
5 +from typing import Any
6 +
7 +from run_ci_command import (
8 + COMMAND_RETRY_FAILED,
9 + COMMAND_RUN_CI,
10 + RETRY_STATES,
11 + BuildkiteClient,
12 + authorize,
13 + create_build_payload,
14 + has_trusted_approval,
15 + is_active_build,
16 + is_build_for_pr,
17 + parse_command,
18 + parse_trusted_users,
19 + run,
20 + select_latest_build,
21 +)
22 +
23 +
24 +def make_pr(**overrides: Any) -> dict[str, Any]:
25 + pr = {
26 + "base": {"ref": "main"},
27 + "draft": False,
28 + "head": {
29 + "ref": "feature",
30 + "repo": {"clone_url": "https://github.com/contributor/vllm.git"},
31 + "sha": "0123456789abcdef",
32 + },
33 + "labels": [],
34 + "number": 42,
35 + "state": "open",
36 + "user": {"login": "author"},
37 + }
38 + pr.update(overrides)
39 + return pr
40 +
41 +
42 +def make_event(command: str, actor: str = "reviewer") -> dict[str, Any]:
43 + return {
44 + "comment": {
45 + "body": command,
46 + "id": 99,
47 + "user": {"login": actor},
48 + },
49 + "issue": {
50 + "number": 42,
51 + "pull_request": {},
52 + },
53 + }
54 +
55 +
56 +class FakeGitHub:
57 + def __init__(
58 + self,
59 + *,
60 + permission: str = "write",
61 + permissions: dict[str, str] | None = None,
62 + pr: dict[str, Any] | None = None,
63 + review_decision: str = "REVIEW_REQUIRED",
64 + reviews: list[dict[str, Any]] | None = None,
65 + ) -> None:
66 + self.comments: list[str] = []
67 + self.permission = permission
68 + self.permissions = permissions or {}
69 + self.pr = pr or make_pr()
70 + self.reactions: list[str] = []
71 + self.review_decision = review_decision
72 + self.reviews = reviews or []
73 +
74 + def get_pr(self, number: int) -> dict[str, Any]:
75 + return self.pr
76 +
77 + def get_permission(self, actor: str) -> str:
78 + return self.permissions.get(actor, self.permission)
79 +
80 + def get_review_decision(self, number: int) -> str:
81 + return self.review_decision
82 +
83 + def list_reviews(self, number: int) -> list[dict[str, Any]]:
84 + return self.reviews
85 +
86 + def list_reactions(self, comment_id: int) -> list[dict[str, Any]]:
87 + return []
88 +
89 + def add_reaction(self, comment_id: int, content: str) -> None:
90 + self.reactions.append(content)
91 +
92 + def add_comment(self, issue_number: int, body: str) -> None:
93 + self.comments.append(body)
94 +
95 +
96 +class FakeBuildkite:
97 + def __init__(
98 + self,
99 + build_lists: list[list[dict[str, Any]]] | None = None,
100 + ) -> None:
101 + self.build_lists = build_lists or []
102 + self.created_builds: list[dict[str, Any]] = []
103 + self.list_calls: list[tuple[str, tuple[str, str] | None]] = []
104 + self.retry_calls: list[tuple[int, str]] = []
105 +
106 + def list_builds(
107 + self,
108 + commit: str,
109 + *,
110 + metadata: tuple[str, str] | None = None,
111 + ) -> list[dict[str, Any]]:
112 + self.list_calls.append((commit, metadata))
113 + return self.build_lists.pop(0)
114 +
115 + def create_build(self, body: dict[str, Any]) -> dict[str, Any]:
116 + self.created_builds.append(body)
117 + return {
118 + "number": 123,
119 + "web_url": "https://buildkite.example/builds/123",
120 + }
121 +
122 + def retry_failed_jobs(
123 + self,
124 + build_number: int,
125 + states: str,
126 + ) -> dict[str, Any]:
127 + self.retry_calls.append((build_number, states))
128 + return {"retried_jobs_count": 3}
129 +
130 +
131 +class FakeTransport:
132 + def __init__(self, response: Any) -> None:
133 + self.calls: list[dict[str, Any]] = []
134 + self.response = response
135 +
136 + def request(self, url: str, **kwargs: Any) -> Any:
137 + self.calls.append({"url": url, **kwargs})
138 + return self.response
139 +
140 +
141 +class RunCiCommandTest(unittest.TestCase):
142 + def test_only_exact_ci_commands_are_accepted(self) -> None:
143 + self.assertEqual(parse_command(COMMAND_RUN_CI), COMMAND_RUN_CI)
144 + self.assertEqual(
145 + parse_command(COMMAND_RETRY_FAILED),
146 + COMMAND_RETRY_FAILED,
147 + )
148 + self.assertIsNone(parse_command("/ci run please"))
149 + self.assertIsNone(parse_command(" /ci run"))
150 +
151 + def test_write_access_authorizes_reviewers_and_authors(self) -> None:
152 + allowed, _ = authorize(
153 + actor="reviewer",
154 + permission="write",
155 + pr=make_pr(),
156 + )
157 + self.assertTrue(allowed)
158 +
159 + def test_configured_trusted_contributors_can_run_ci(self) -> None:
160 + trusted_users = parse_trusted_users("trusted-one, TRUSTED-TWO")
161 + allowed, _ = authorize(
162 + actor="trusted-two",
163 + permission="read",
164 + pr=make_pr(),
165 + trusted_users=trusted_users,
166 + )
167 + self.assertTrue(allowed)
168 +
169 + def test_authors_need_an_approval_or_ready_label(self) -> None:
170 + pending, _ = authorize(
171 + actor="author",
172 + permission="read",
173 + pr=make_pr(),
174 + )
175 + approved, _ = authorize(
176 + actor="author",
177 + permission="read",
178 + pr=make_pr(),
179 + trusted_approval=True,
180 + )
181 + ready, _ = authorize(
182 + actor="author",
183 + permission="read",
184 + pr=make_pr(labels=[{"name": "ready"}]),
185 + )
186 + self.assertFalse(pending)
187 + self.assertTrue(approved)
188 + self.assertTrue(ready)
189 +
190 + def test_non_author_contributors_without_write_are_denied(self) -> None:
191 + allowed, _ = authorize(
192 + actor="contributor",
193 + permission="read",
194 + pr=make_pr(),
195 + trusted_approval=True,
196 + )
197 + self.assertFalse(allowed)
198 +
199 + def test_authors_cannot_use_ready_state_on_draft_prs(self) -> None:
200 + allowed, _ = authorize(
201 + actor="author",
202 + permission="read",
203 + pr=make_pr(draft=True, labels=[{"name": "ready"}]),
204 + trusted_approval=True,
205 + )
206 + self.assertFalse(allowed)
207 +
208 + def test_only_trusted_reviewers_can_delegate_through_approval(self) -> None:
209 + approved_review = {
210 + "state": "APPROVED",
211 + "user": {"login": "reviewer"},
212 + }
213 + trusted = FakeGitHub(
214 + permission="read",
215 + permissions={"reviewer": "write"},
216 + review_decision="APPROVED",
217 + reviews=[approved_review],
218 + )
219 + untrusted = FakeGitHub(
220 + permission="read",
221 + review_decision="APPROVED",
222 + reviews=[approved_review],
223 + )
224 + self.assertTrue(has_trusted_approval(trusted, 42, set()))
225 + self.assertFalse(has_trusted_approval(untrusted, 42, set()))
226 +
227 + def test_build_matching_is_scoped_to_the_pr(self) -> None:
228 + self.assertTrue(is_build_for_pr({"pull_request": {"id": 42}}, 42))
229 + self.assertFalse(is_build_for_pr({"pull_request": {"id": 43}}, 42))
230 + self.assertTrue(
231 + is_build_for_pr(
232 + {"meta_data": {"github-pr-number": "42"}},
233 + 42,
234 + )
235 + )
236 +
237 + def test_latest_build_selection_ignores_other_prs(self) -> None:
238 + latest = select_latest_build(
239 + [
240 + {
241 + "created_at": "2026-07-28T02:00:00Z",
242 + "number": 3,
243 + "pull_request": {"id": 43},
244 + },
245 + {
246 + "created_at": "2026-07-28T01:00:00Z",
247 + "number": 2,
248 + "pull_request": {"id": 42},
249 + },
250 + {
251 + "created_at": "2026-07-28T00:00:00Z",
252 + "number": 1,
253 + "pull_request": {"id": 42},
254 + },
255 + ],
256 + 42,
257 + )
258 + self.assertEqual(latest["number"], 2)
259 +
260 + def test_active_build_states_prevent_duplicate_runs(self) -> None:
261 + self.assertTrue(is_active_build({"state": "scheduled"}))
262 + self.assertTrue(is_active_build({"state": "running"}))
263 + self.assertTrue(is_active_build({"state": "waiting"}))
264 + self.assertTrue(is_active_build({"blocked": True, "state": "passed"}))
265 + self.assertFalse(is_active_build({"state": "failed"}))
266 +
267 + def test_build_payload_preserves_pr_context(self) -> None:
268 + payload = create_build_payload(
269 + actor="reviewer",
270 + comment_id=99,
271 + pr=make_pr(labels=[{"name": "ready"}, {"name": "v1"}]),
272 + )
273 + self.assertEqual(
274 + payload,
275 + {
276 + "commit": "0123456789abcdef",
277 + "branch": "feature",
278 + "message": "PR #42 /ci run by @reviewer",
279 + "pull_request_id": 42,
280 + "pull_request_base_branch": "main",
281 + "pull_request_repository": ("https://github.com/contributor/vllm.git"),
282 + "pull_request_labels": ["ready", "v1"],
283 + "env": {
284 + "VLLM_CI_GITHUB_COMMENT_ID": "99",
285 + "VLLM_CI_TRIGGERED_BY": "reviewer",
286 + },
287 + "meta_data": {
288 + "github-comment-id": "99",
289 + "github-pr-number": "42",
290 + "github-triggered-by": "reviewer",
291 + },
292 + },
293 + )
294 +
295 + def test_ci_run_dispatches_build_with_current_pr_metadata(self) -> None:
296 + github = FakeGitHub()
297 + buildkite = FakeBuildkite([[], []])
298 + run(make_event(COMMAND_RUN_CI), github, buildkite)
299 +
300 + self.assertEqual(len(buildkite.created_builds), 1)
301 + self.assertEqual(
302 + buildkite.created_builds[0]["message"],
303 + "PR #42 /ci run by @reviewer",
304 + )
305 + self.assertEqual(github.reactions, ["eyes", "rocket"])
306 + self.assertIn("Buildkite CI #123", github.comments[0])
307 +
308 + def test_unapproved_authors_are_denied_without_buildkite(self) -> None:
309 + github = FakeGitHub(
310 + permission="read",
311 + pr=make_pr(),
312 + review_decision="REVIEW_REQUIRED",
313 + )
314 + buildkite = FakeBuildkite()
315 + run(make_event(COMMAND_RUN_CI, "author"), github, buildkite)
316 +
317 + self.assertEqual(buildkite.list_calls, [])
318 + self.assertEqual(github.reactions, ["eyes", "-1"])
319 + self.assertIn("approve the PR", github.comments[0])
320 +
321 + def test_ci_retry_uses_latest_current_sha_build(self) -> None:
322 + github = FakeGitHub(
323 + permission="read",
324 + pr=make_pr(labels=[{"name": "ready"}]),
325 + )
326 + buildkite = FakeBuildkite(
327 + [
328 + [
329 + {
330 + "created_at": "2026-07-28T01:00:00Z",
331 + "finished_at": "2026-07-28T02:00:00Z",
332 + "number": 123,
333 + "pull_request": {"id": 42},
334 + "state": "failed",
335 + "web_url": "https://buildkite.example/builds/123",
336 + }
337 + ]
338 + ]
339 + )
340 + run(make_event(COMMAND_RETRY_FAILED, "author"), github, buildkite)
341 +
342 + self.assertEqual(buildkite.retry_calls, [(123, RETRY_STATES)])
343 + self.assertIn("Queued 3 failed job", github.comments[0])
344 +
345 + def test_buildkite_retry_uses_retry_failed_jobs_endpoint(self) -> None:
346 + transport = FakeTransport({"retried_jobs_count": 2})
347 + client = BuildkiteClient(
348 + "secret",
349 + "vllm",
350 + "ci",
351 + transport=transport,
352 + )
353 + client.retry_failed_jobs(123, RETRY_STATES)
354 +
355 + call = transport.calls[0]
356 + self.assertEqual(call["method"], "PUT")
357 + self.assertTrue(call["url"].endswith("/123/retry_failed_jobs"))
358 + self.assertEqual(call["body"], {"states": RETRY_STATES})
359 +
360 +
361 +if __name__ == "__main__":
362 + unittest.main()
docs/contributing/README.md
+4 -2
@@ -301,8 +301,10 @@ review process:
301 isn't clear or you disagree with a suggestion, feel free to ask for
302 clarification or discuss the suggestion.
303 - Note that not all CI checks will be executed due to limited computational
304 - resources. The reviewer will add `ready` label to the PR when the PR is
305 - ready to merge or a full CI run is needed.
304 + resources. Reviewers with write access and configured trusted contributors
305 + can comment `/ci run` when CI signals are needed before a PR is ready. After
306 + the PR is approved or has the `ready` label, the PR author can use `/ci run`
307 + or `/ci retry`. New commits do not start CI automatically.
308
309 ### Pull Request Limits and Escalation
310