[CI] Retry Buildkite API rate limits (#50481)
Signed-off-by: khluu <khluu000@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com>
Kevin H. Luu committed
Jul 30, 2026 at 19:09 UTC
60399d48eb67785dd130fa8db13debbcf310e2ed
2 files changed
+178
-13
.github/workflows/scripts/run_ci_command.py
+72
-13
@@ -3,11 +3,13 @@
3
4
import json
5
import os
6
+import random
7
import sys
8
+import time
9
import urllib.error
10
import urllib.parse
11
import urllib.request
10
-from collections.abc import Mapping, Sequence
12
+from collections.abc import Callable, Mapping, Sequence
13
from typing import Any
14
15
COMMAND_RUN_CI = "/ci run"
@@ -39,7 +41,22 @@ class ApiError(RuntimeError):
41
self.status = status
42
43
44
+def rate_limit_jitter() -> float:
45
+ return random.uniform(1, 5)
46
+
47
+
48
class HttpTransport:
49
+ def __init__(
50
+ self,
51
+ *,
52
+ max_retries: int = 3,
53
+ jitter: Callable[[], float] = rate_limit_jitter,
54
+ sleep: Callable[[float], None] = time.sleep,
55
+ ) -> None:
56
+ self.max_retries = max_retries
57
+ self.jitter = jitter
58
+ self.sleep = sleep
59
+
60
def request(
61
self,
62
url: str,
@@ -55,18 +72,33 @@ class HttpTransport:
72
headers=dict(headers or {}),
73
method=method,
74
)
58
- try:
59
- with urllib.request.urlopen(request, timeout=30) as response:
60
- response_body = response.read().decode()
61
- except urllib.error.HTTPError as error:
62
- response_body = error.read().decode()
63
- message = self._error_message(response_body, error.reason)
64
- raise ApiError(
65
- error.code,
66
- f"API returned {error.code}: {message}",
67
- ) from error
68
- except urllib.error.URLError as error:
69
- raise ApiError(None, f"API request failed: {error.reason}") from error
75
+ for attempt in range(self.max_retries + 1):
76
+ try:
77
+ with urllib.request.urlopen(request, timeout=30) as response:
78
+ response_body = response.read().decode()
79
+ break
80
+ except urllib.error.HTTPError as error:
81
+ response_body = error.read().decode()
82
+ if error.code == 429 and attempt < self.max_retries:
83
+ delay = self._rate_limit_delay(error, response_body)
84
+ print(
85
+ "API rate limit reached; "
86
+ f"retry {attempt + 1}/{self.max_retries} "
87
+ f"in {delay:g} seconds.",
88
+ file=sys.stderr,
89
+ )
90
+ self.sleep(delay)
91
+ continue
92
+ message = self._error_message(response_body, error.reason)
93
+ raise ApiError(
94
+ error.code,
95
+ f"API returned {error.code}: {message}",
96
+ ) from error
97
+ except urllib.error.URLError as error:
98
+ raise ApiError(
99
+ None,
100
+ f"API request failed: {error.reason}",
101
+ ) from error
102
103
if not response_body:
104
return None
@@ -83,6 +115,33 @@ class HttpTransport:
115
return fallback
116
return str(parsed.get("message", fallback))
117
118
+ def _rate_limit_delay(
119
+ self,
120
+ error: urllib.error.HTTPError,
121
+ response_body: str,
122
+ ) -> float:
123
+ try:
124
+ parsed = json.loads(response_body)
125
+ except json.JSONDecodeError:
126
+ parsed = {}
127
+
128
+ scope = parsed.get("scope")
129
+ reset_header = (
130
+ "RateLimit-User-Reset" if scope == "rest_user" else "RateLimit-Reset"
131
+ )
132
+ candidates = [
133
+ error.headers.get(reset_header),
134
+ parsed.get("reset"),
135
+ ]
136
+ for candidate in candidates:
137
+ try:
138
+ delay = float(candidate)
139
+ except (TypeError, ValueError):
140
+ continue
141
+ if delay >= 0:
142
+ return delay + self.jitter()
143
+ return 60 + self.jitter()
144
+
145
146
class GitHubClient:
147
def __init__(
.github/workflows/scripts/test_run_ci_command.py
+106
@@ -1,9 +1,12 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
+import io
5
import json
6
import unittest
7
+import urllib.error
8
from typing import Any
9
+from unittest.mock import patch
10
11
from run_ci_command import (
12
CI_AUTHORIZED_COMMENT_MARKER,
@@ -11,6 +14,7 @@ from run_ci_command import (
14
COMMAND_RUN_CI,
15
RETRY_STATES,
16
BuildkiteClient,
17
+ HttpTransport,
18
authorize,
19
create_build_payload,
20
has_trusted_approval,
@@ -160,7 +164,109 @@ class FakeTransport:
164
return self.response
165
166
167
+class FakeHttpResponse:
168
+ def __init__(self, response: Any) -> None:
169
+ self.response = response
170
+
171
+ def __enter__(self) -> "FakeHttpResponse":
172
+ return self
173
+
174
+ def __exit__(self, *args: Any) -> None:
175
+ return None
176
+
177
+ def read(self) -> bytes:
178
+ return json.dumps(self.response).encode()
179
+
180
+
181
class RunCiCommandTest(unittest.TestCase):
182
+ @patch("run_ci_command.urllib.request.urlopen")
183
+ def test_http_transport_retries_buildkite_rate_limit(self, urlopen: Any) -> None:
184
+ body = {
185
+ "message": "Please wait 9 seconds before making more requests.",
186
+ "reset": 9,
187
+ "scope": "rest",
188
+ }
189
+ rate_limit_error = urllib.error.HTTPError(
190
+ "https://api.buildkite.com/v2/builds",
191
+ 429,
192
+ "Too Many Requests",
193
+ {
194
+ "RateLimit-Limit": "400",
195
+ "RateLimit-Remaining": "0",
196
+ "RateLimit-Reset": "9",
197
+ },
198
+ io.BytesIO(json.dumps(body).encode()),
199
+ )
200
+ urlopen.side_effect = [rate_limit_error, FakeHttpResponse({"ok": True})]
201
+ delays: list[float] = []
202
+
203
+ response = HttpTransport(
204
+ jitter=lambda: 2.5,
205
+ sleep=delays.append,
206
+ ).request("https://api.buildkite.com/v2/builds")
207
+
208
+ self.assertEqual(response, {"ok": True})
209
+ self.assertEqual(delays, [11.5])
210
+ self.assertEqual(urlopen.call_count, 2)
211
+
212
+ @patch("run_ci_command.urllib.request.urlopen")
213
+ def test_http_transport_retries_rate_limit_three_times(self, urlopen: Any) -> None:
214
+ def rate_limit_error() -> urllib.error.HTTPError:
215
+ body = {
216
+ "message": "Please wait 9 seconds before making more requests.",
217
+ "reset": 9,
218
+ "scope": "rest",
219
+ }
220
+ return urllib.error.HTTPError(
221
+ "https://api.buildkite.com/v2/builds",
222
+ 429,
223
+ "Too Many Requests",
224
+ {
225
+ "RateLimit-Limit": "400",
226
+ "RateLimit-Remaining": "0",
227
+ "RateLimit-Reset": "9",
228
+ },
229
+ io.BytesIO(json.dumps(body).encode()),
230
+ )
231
+
232
+ urlopen.side_effect = [rate_limit_error() for _ in range(4)]
233
+ delays: list[float] = []
234
+
235
+ with self.assertRaisesRegex(RuntimeError, "API returned 429"):
236
+ HttpTransport(
237
+ jitter=lambda: 2,
238
+ sleep=delays.append,
239
+ ).request("https://api.buildkite.com/v2/builds")
240
+
241
+ self.assertEqual(delays, [11, 11, 11])
242
+ self.assertEqual(urlopen.call_count, 4)
243
+
244
+ @patch("run_ci_command.urllib.request.urlopen")
245
+ def test_http_transport_does_not_retry_permission_error(self, urlopen: Any) -> None:
246
+ permission_error = urllib.error.HTTPError(
247
+ "https://api.github.com/repos/vllm-project/vllm/issues/1/comments",
248
+ 403,
249
+ "Forbidden",
250
+ {},
251
+ io.BytesIO(b'{"message":"Resource not accessible by integration"}'),
252
+ )
253
+ urlopen.side_effect = permission_error
254
+ delays: list[float] = []
255
+
256
+ with self.assertRaisesRegex(
257
+ RuntimeError,
258
+ "Resource not accessible by integration",
259
+ ):
260
+ HttpTransport(
261
+ jitter=lambda: 2,
262
+ sleep=delays.append,
263
+ ).request(
264
+ "https://api.github.com/repos/vllm-project/vllm/issues/1/comments"
265
+ )
266
+
267
+ self.assertEqual(delays, [])
268
+ self.assertEqual(urlopen.call_count, 1)
269
+
270
def test_only_exact_ci_commands_are_accepted(self) -> None:
271
self.assertEqual(parse_command(COMMAND_RUN_CI), COMMAND_RUN_CI)
272
self.assertEqual(