[CI] Retry failed steps on new PR commits (#50318)
Signed-off-by: khluu <khluu000@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com>
Kevin H. Luu committed
Jul 29, 2026 at 23:54 UTC
445d3aac0a0720f38e80c256c884f4aa45cb8ab8
2 files changed
+376
-26
.github/workflows/scripts/run_ci_command.py
+185
-24
@@ -25,6 +25,11 @@ ACTIVE_BUILD_STATES = {
25
"waiting_failed",
26
}
27
RETRY_STATES = "failed,timed_out,expired"
28
+SETUP_STEP_KEYS = {
29
+ "ensure-ci-base-amd",
30
+ "pre-commit",
31
+ "refresh-rocm-base-amd",
32
+}
33
34
35
class ApiError(RuntimeError):
@@ -207,6 +212,35 @@ class BuildkiteClient:
212
f"{organization}/pipelines/{pipeline}/builds"
213
)
214
215
+ def _headers(self) -> dict[str, str]:
216
+ if not self.token:
217
+ raise RuntimeError("The BUILDKITE_API_TOKEN repository secret is not set.")
218
+ return {
219
+ "Authorization": f"Bearer {self.token}",
220
+ "Content-Type": "application/json",
221
+ "User-Agent": "vllm-ci-command",
222
+ }
223
+
224
+ def _request_url(
225
+ self,
226
+ url: str,
227
+ *,
228
+ body: Mapping[str, Any] | None = None,
229
+ method: str = "GET",
230
+ ) -> Any:
231
+ if (
232
+ url != self.base_url
233
+ and not url.startswith(f"{self.base_url}?")
234
+ and not url.startswith(f"{self.base_url}/")
235
+ ):
236
+ raise ApiError(None, "Buildkite API returned an invalid pagination URL.")
237
+ return self.transport.request(
238
+ url,
239
+ body=body,
240
+ headers=self._headers(),
241
+ method=method,
242
+ )
243
+
244
def _request(
245
self,
246
*,
@@ -215,34 +249,24 @@ class BuildkiteClient:
249
path: str = "",
250
query: Sequence[tuple[str, str]] = (),
251
) -> Any:
218
- if not self.token:
219
- raise RuntimeError("The BUILDKITE_API_TOKEN repository secret is not set.")
252
url = f"{self.base_url}{path}"
253
if query:
254
url = f"{url}?{urllib.parse.urlencode(query)}"
223
- return self.transport.request(
224
- url,
225
- body=body,
226
- headers={
227
- "Authorization": f"Bearer {self.token}",
228
- "Content-Type": "application/json",
229
- "User-Agent": "vllm-ci-command",
230
- },
231
- method=method,
232
- )
255
+ return self._request_url(url, body=body, method=method)
256
257
def list_builds(
258
self,
236
- commit: str,
259
+ commit: str | None,
260
*,
261
metadata: tuple[str, str] | None = None,
262
) -> list[dict[str, Any]]:
263
query = [
241
- ("commit", commit),
264
("exclude_jobs", "true"),
265
("exclude_pipeline", "true"),
266
("per_page", "100"),
267
]
268
+ if commit:
269
+ query.append(("commit", commit))
270
if metadata:
271
key, value = metadata
272
query.append((f"meta_data[{key}]", value))
@@ -266,6 +290,34 @@ class BuildkiteClient:
290
path=f"/{number}/retry_failed_jobs",
291
)
292
293
+ def list_failed_jobs(self, build_number: int) -> list[dict[str, Any]]:
294
+ number = urllib.parse.quote(str(build_number), safe="")
295
+ query = [
296
+ ("state[]", "failed"),
297
+ ("state[]", "timed_out"),
298
+ ("state[]", "expired"),
299
+ ("include_retried_jobs", "false"),
300
+ ("per_page", "100"),
301
+ ]
302
+ url = f"{self.base_url}/{number}/jobs?{urllib.parse.urlencode(query)}"
303
+ jobs: list[dict[str, Any]] = []
304
+ while url:
305
+ response = self._request_url(url)
306
+ if not isinstance(response, Mapping):
307
+ raise ApiError(None, "Buildkite API returned an invalid job list.")
308
+ items = response.get("items")
309
+ links = response.get("links")
310
+ if not isinstance(items, list) or not isinstance(links, Mapping):
311
+ raise ApiError(None, "Buildkite API returned an invalid job list.")
312
+ jobs.extend(items)
313
+ next_url = links.get("next")
314
+ if next_url is not None and not isinstance(next_url, str):
315
+ raise ApiError(
316
+ None, "Buildkite API returned an invalid pagination URL."
317
+ )
318
+ url = next_url
319
+ return jobs
320
+
321
322
def parse_command(body: str) -> str | None:
323
if body in {COMMAND_RUN_CI, COMMAND_RETRY_FAILED}:
@@ -394,6 +446,33 @@ def create_build_payload(
446
}
447
448
449
+def create_retry_build_payload(
450
+ *,
451
+ actor: str,
452
+ comment_id: int,
453
+ pr: Mapping[str, Any],
454
+ source_build: Mapping[str, Any],
455
+ step_keys: Sequence[str],
456
+) -> dict[str, Any]:
457
+ payload = create_build_payload(
458
+ actor=actor,
459
+ comment_id=comment_id,
460
+ pr=pr,
461
+ )
462
+ source_number = str(source_build["number"])
463
+ payload["message"] = f"PR #{pr['number']} {COMMAND_RETRY_FAILED} by @{actor}"
464
+ payload["env"]["VLLM_CI_ONLY_STEP_KEYS"] = json.dumps(
465
+ step_keys, separators=(",", ":")
466
+ )
467
+ payload["meta_data"].update(
468
+ {
469
+ "github-retry-source-build": source_number,
470
+ "github-retry-source-commit": str(source_build.get("commit", "")),
471
+ }
472
+ )
473
+ return payload
474
+
475
+
476
def add_reaction_safely(
477
github: GitHubClient,
478
comment_id: int,
@@ -462,24 +541,100 @@ def handle_run_ci(
541
542
def handle_retry_failed(
543
*,
544
+ actor: str,
545
buildkite: BuildkiteClient,
546
+ comment_id: int,
547
+ github: GitHubClient,
548
pr: Mapping[str, Any],
549
) -> str:
550
builds = buildkite.list_builds(pr["head"]["sha"])
551
build = select_latest_build(builds, pr["number"])
470
- if not build:
471
- return "No CI build exists for the current PR commit. Use `/ci run` first."
472
- if not build.get("finished_at") or is_active_build(build):
473
- return f"CI is still running for this commit: {build['web_url']}"
552
+ if build:
553
+ metadata = build.get("meta_data") or {}
554
+ if str(metadata.get("github-comment-id")) == str(comment_id):
555
+ return f"CI was already requested by this comment: {build['web_url']}"
556
+ if not build.get("finished_at") or is_active_build(build):
557
+ return f"CI is still running for this commit: {build['web_url']}"
558
+
559
+ retried = buildkite.retry_failed_jobs(build["number"], RETRY_STATES)
560
+ if retried["retried_jobs_count"] == 0:
561
+ return (
562
+ "No failed, timed-out, or expired jobs need retrying: "
563
+ f"{build['web_url']}"
564
+ )
565
+ return (
566
+ f"Queued {retried['retried_jobs_count']} failed job(s) for retry in "
567
+ f"[Buildkite CI #{build['number']}]({build['web_url']})."
568
+ )
569
475
- retried = buildkite.retry_failed_jobs(build["number"], RETRY_STATES)
476
- if retried["retried_jobs_count"] == 0:
570
+ previous_builds = buildkite.list_builds(
571
+ None,
572
+ metadata=("github-pr-number", str(pr["number"])),
573
+ )
574
+ previous_builds = [
575
+ candidate
576
+ for candidate in previous_builds
577
+ if candidate.get("commit") != pr["head"]["sha"]
578
+ ]
579
+ source_build = select_latest_build(previous_builds, pr["number"])
580
+ if not source_build:
581
+ return "No earlier CI build exists for this PR. Use `/ci run` first."
582
+ if not source_build.get("finished_at") or is_active_build(source_build):
583
+ return f"The previous CI build is still running: {source_build['web_url']}"
584
+
585
+ failed_jobs = buildkite.list_failed_jobs(source_build["number"])
586
+ failed_script_jobs = [job for job in failed_jobs if job.get("type") == "script"]
587
+ missing_step_keys = [job for job in failed_script_jobs if not job.get("step_key")]
588
+ if missing_step_keys:
589
return (
478
- f"No failed, timed-out, or expired jobs need retrying: {build['web_url']}"
590
+ f"[Buildkite CI #{source_build['number']}]"
591
+ f"({source_build['web_url']}) has failed jobs without stable step "
592
+ "keys, so they cannot be retried on a new commit. Use `/ci run`."
593
)
594
+
595
+ failed_step_keys = {str(job["step_key"]) for job in failed_script_jobs}
596
+ setup_failures = sorted(
597
+ step_key
598
+ for step_key in failed_step_keys
599
+ if step_key.startswith("image-build") or step_key in SETUP_STEP_KEYS
600
+ )
601
+ if setup_failures:
602
+ return (
603
+ f"[Buildkite CI #{source_build['number']}]"
604
+ f"({source_build['web_url']}) failed during CI setup, so its test "
605
+ "failure set is incomplete. Use `/ci run` for the new commit."
606
+ )
607
+
608
+ step_keys = sorted(failed_step_keys)
609
+ if not step_keys:
610
+ return (
611
+ "No failed, timed-out, or expired jobs need retrying in "
612
+ f"[Buildkite CI #{source_build['number']}]"
613
+ f"({source_build['web_url']})."
614
+ )
615
+
616
+ current_pr = github.get_pr(pr["number"])
617
+ if current_pr["state"] != "open" or current_pr["head"]["sha"] != pr["head"]["sha"]:
618
+ return (
619
+ "The PR head changed while processing the command. "
620
+ "Comment `/ci retry` again."
621
+ )
622
+
623
+ retry_build = buildkite.create_build(
624
+ create_retry_build_payload(
625
+ actor=actor,
626
+ comment_id=comment_id,
627
+ pr=current_pr,
628
+ source_build=source_build,
629
+ step_keys=step_keys,
630
+ )
631
+ )
632
return (
481
- f"Queued {retried['retried_jobs_count']} failed job(s) for retry in "
482
- f"[Buildkite CI #{build['number']}]({build['web_url']})."
633
+ f"Triggered [Buildkite CI #{retry_build['number']}]"
634
+ f"({retry_build['web_url']}) for commit "
635
+ f"`{current_pr['head']['sha'][:12]}`, running {len(step_keys)} failed "
636
+ f"step(s) from [Buildkite CI #{source_build['number']}]"
637
+ f"({source_build['web_url']})."
638
)
639
640
@@ -544,7 +699,13 @@ def run(
699
pr=pr,
700
)
701
else:
547
- message = handle_retry_failed(buildkite=buildkite, pr=pr)
702
+ message = handle_retry_failed(
703
+ actor=actor,
704
+ buildkite=buildkite,
705
+ comment_id=comment_id,
706
+ github=github,
707
+ pr=pr,
708
+ )
709
add_reaction_safely(github, comment_id, "rocket")
710
github.add_comment(issue_number, message)
711
except Exception:
.github/workflows/scripts/test_run_ci_command.py
+191
-2
@@ -1,6 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
+import json
5
import unittest
6
from typing import Any
7
@@ -97,15 +98,18 @@ class FakeBuildkite:
98
def __init__(
99
self,
100
build_lists: list[list[dict[str, Any]]] | None = None,
101
+ failed_job_lists: list[list[dict[str, Any]]] | None = None,
102
) -> None:
103
self.build_lists = build_lists or []
104
self.created_builds: list[dict[str, Any]] = []
103
- self.list_calls: list[tuple[str, tuple[str, str] | None]] = []
105
+ self.failed_job_lists = failed_job_lists or []
106
+ self.job_list_calls: list[int] = []
107
+ self.list_calls: list[tuple[str | None, tuple[str, str] | None]] = []
108
self.retry_calls: list[tuple[int, str]] = []
109
110
def list_builds(
111
self,
108
- commit: str,
112
+ commit: str | None,
113
*,
114
metadata: tuple[str, str] | None = None,
115
) -> list[dict[str, Any]]:
@@ -127,6 +131,10 @@ class FakeBuildkite:
131
self.retry_calls.append((build_number, states))
132
return {"retried_jobs_count": 3}
133
134
+ def list_failed_jobs(self, build_number: int) -> list[dict[str, Any]]:
135
+ self.job_list_calls.append(build_number)
136
+ return self.failed_job_lists.pop(0)
137
+
138
139
class FakeTransport:
140
def __init__(self, response: Any) -> None:
@@ -135,6 +143,8 @@ class FakeTransport:
143
144
def request(self, url: str, **kwargs: Any) -> Any:
145
self.calls.append({"url": url, **kwargs})
146
+ if isinstance(self.response, tuple):
147
+ return self.response[len(self.calls) - 1]
148
return self.response
149
150
@@ -343,6 +353,134 @@ class RunCiCommandTest(unittest.TestCase):
353
self.assertEqual(buildkite.retry_calls, [(123, RETRY_STATES)])
354
self.assertIn("Queued 3 failed job", github.comments[0])
355
356
+ def test_ci_retry_creates_filtered_build_for_new_head(self) -> None:
357
+ github = FakeGitHub(
358
+ permission="read",
359
+ pr=make_pr(labels=[{"name": "ready"}]),
360
+ )
361
+ source_build = {
362
+ "commit": "old-commit",
363
+ "created_at": "2026-07-28T01:00:00Z",
364
+ "finished_at": "2026-07-28T02:00:00Z",
365
+ "number": 122,
366
+ "pull_request": {"id": 42},
367
+ "state": "failed",
368
+ "web_url": "https://buildkite.example/builds/122",
369
+ }
370
+ buildkite = FakeBuildkite(
371
+ [[], [source_build]],
372
+ [
373
+ [
374
+ {
375
+ "state": "failed",
376
+ "step_key": "basic-models-test-other-cpu",
377
+ "type": "script",
378
+ },
379
+ {
380
+ "state": "failed",
381
+ "step_key": "basic-models-test-other-cpu",
382
+ "type": "script",
383
+ },
384
+ {
385
+ "state": "timed_out",
386
+ "step_key": "distributed-tests-2xh100-2xmi300",
387
+ "type": "script",
388
+ },
389
+ ]
390
+ ],
391
+ )
392
+
393
+ run(make_event(COMMAND_RETRY_FAILED, "author"), github, buildkite)
394
+
395
+ self.assertEqual(buildkite.job_list_calls, [122])
396
+ self.assertEqual(len(buildkite.created_builds), 1)
397
+ payload = buildkite.created_builds[0]
398
+ self.assertEqual(payload["commit"], "0123456789abcdef")
399
+ self.assertEqual(payload["message"], "PR #42 /ci retry by @author")
400
+ self.assertEqual(
401
+ json.loads(payload["env"]["VLLM_CI_ONLY_STEP_KEYS"]),
402
+ [
403
+ "basic-models-test-other-cpu",
404
+ "distributed-tests-2xh100-2xmi300",
405
+ ],
406
+ )
407
+ self.assertEqual(
408
+ payload["meta_data"]["github-retry-source-build"],
409
+ "122",
410
+ )
411
+ self.assertEqual(
412
+ payload["meta_data"]["github-retry-source-commit"],
413
+ "old-commit",
414
+ )
415
+ self.assertIn("running 2 failed step", github.comments[0])
416
+ self.assertIn("Buildkite CI #122", github.comments[0])
417
+
418
+ def test_ci_retry_new_head_requires_stable_step_keys(self) -> None:
419
+ github = FakeGitHub(
420
+ permission="read",
421
+ pr=make_pr(labels=[{"name": "ready"}]),
422
+ )
423
+ buildkite = FakeBuildkite(
424
+ [
425
+ [],
426
+ [
427
+ {
428
+ "commit": "old-commit",
429
+ "created_at": "2026-07-28T01:00:00Z",
430
+ "finished_at": "2026-07-28T02:00:00Z",
431
+ "number": 122,
432
+ "pull_request": {"id": 42},
433
+ "state": "failed",
434
+ "web_url": "https://buildkite.example/builds/122",
435
+ }
436
+ ],
437
+ ],
438
+ [[{"state": "failed", "step_key": None, "type": "script"}]],
439
+ )
440
+
441
+ run(make_event(COMMAND_RETRY_FAILED, "author"), github, buildkite)
442
+
443
+ self.assertEqual(buildkite.created_builds, [])
444
+ self.assertIn("without stable step keys", github.comments[0])
445
+ self.assertIn("Use `/ci run`", github.comments[0])
446
+
447
+ def test_ci_retry_new_head_rejects_incomplete_setup_failure(self) -> None:
448
+ github = FakeGitHub(
449
+ permission="read",
450
+ pr=make_pr(labels=[{"name": "ready"}]),
451
+ )
452
+ buildkite = FakeBuildkite(
453
+ [
454
+ [],
455
+ [
456
+ {
457
+ "commit": "old-commit",
458
+ "created_at": "2026-07-28T01:00:00Z",
459
+ "finished_at": "2026-07-28T02:00:00Z",
460
+ "number": 122,
461
+ "pull_request": {"id": 42},
462
+ "state": "failed",
463
+ "web_url": "https://buildkite.example/builds/122",
464
+ }
465
+ ],
466
+ ],
467
+ [
468
+ [
469
+ {
470
+ "state": "failed",
471
+ "step_key": "image-build",
472
+ "type": "script",
473
+ }
474
+ ]
475
+ ],
476
+ )
477
+
478
+ run(make_event(COMMAND_RETRY_FAILED, "author"), github, buildkite)
479
+
480
+ self.assertEqual(buildkite.created_builds, [])
481
+ self.assertIn("failed during CI setup", github.comments[0])
482
+ self.assertIn("Use `/ci run`", github.comments[0])
483
+
484
def test_buildkite_retry_uses_retry_failed_jobs_endpoint(self) -> None:
485
transport = FakeTransport({"retried_jobs_count": 2})
486
client = BuildkiteClient(
@@ -358,6 +496,57 @@ class RunCiCommandTest(unittest.TestCase):
496
self.assertTrue(call["url"].endswith("/123/retry_failed_jobs"))
497
self.assertEqual(call["body"], {"states": RETRY_STATES})
498
499
+ def test_buildkite_list_builds_allows_query_on_builds_endpoint(self) -> None:
500
+ transport = FakeTransport([])
501
+ client = BuildkiteClient(
502
+ "secret",
503
+ "vllm",
504
+ "ci",
505
+ transport=transport,
506
+ )
507
+
508
+ builds = client.list_builds(
509
+ "current-commit",
510
+ metadata=("github-pr-number", "42"),
511
+ )
512
+
513
+ self.assertEqual(builds, [])
514
+ url = transport.calls[0]["url"]
515
+ self.assertIn("?exclude_jobs=true", url)
516
+ self.assertIn("commit=current-commit", url)
517
+ self.assertIn("meta_data%5Bgithub-pr-number%5D=42", url)
518
+
519
+ def test_buildkite_failed_jobs_follow_cursor_pagination(self) -> None:
520
+ next_url = (
521
+ "https://api.buildkite.com/v2/organizations/vllm/pipelines/ci/"
522
+ "builds/123/jobs?after=cursor"
523
+ )
524
+ transport = FakeTransport(
525
+ (
526
+ {
527
+ "items": [{"id": "first"}],
528
+ "links": {"next": next_url},
529
+ },
530
+ {
531
+ "items": [{"id": "second"}],
532
+ "links": {"next": None},
533
+ },
534
+ )
535
+ )
536
+ client = BuildkiteClient(
537
+ "secret",
538
+ "vllm",
539
+ "ci",
540
+ transport=transport,
541
+ )
542
+
543
+ jobs = client.list_failed_jobs(123)
544
+
545
+ self.assertEqual(jobs, [{"id": "first"}, {"id": "second"}])
546
+ self.assertIn("state%5B%5D=failed", transport.calls[0]["url"])
547
+ self.assertIn("include_retried_jobs=false", transport.calls[0]["url"])
548
+ self.assertEqual(transport.calls[1]["url"], next_url)
549
+
550
551
if __name__ == "__main__":
552
unittest.main()