A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 09:30 UTC
[CI] Add retry with 4x backoff to HTTP fetches for transient failures (#37218)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Andreas Karatzas committed
Mar 19, 2026 at 14:00 UTC
fb8b5e05fcd697ed1e0002a7950cf42c36d77e6b
2 files changed
+230
-18
vllm/connections.py
+223
-18
@@ -1,15 +1,201 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
-from collections.abc import Mapping, MutableMapping
4
+import asyncio
5
+import functools
6
+import time
7
+from collections.abc import Callable, Coroutine, Mapping, MutableMapping
8
from pathlib import Path
9
+from typing import Any, ParamSpec, TypeVar
10
11
import aiohttp
12
import requests
13
from urllib3.util import parse_url
14
15
+import vllm.envs as envs
16
+from vllm.logger import init_logger
17
from vllm.version import __version__ as VLLM_VERSION
18
19
+logger = init_logger(__name__)
20
+
21
+_P = ParamSpec("_P")
22
+_T = TypeVar("_T")
23
+
24
+# Multiplier applied to timeout and sleep on each retry attempt.
25
+# Attempt N uses: base_timeout * (_RETRY_BACKOFF_FACTOR ** N) for the
26
+# per-attempt timeout and sleeps _RETRY_BACKOFF_FACTOR ** N seconds.
27
+_RETRY_BACKOFF_FACTOR = 4
28
+
29
+
30
+def _is_retryable(exc: Exception) -> bool:
31
+ """Return True for transient errors that are worth retrying.
32
+
33
+ Retryable:
34
+ - Timeouts (aiohttp, requests, stdlib)
35
+ - Connection-level failures (refused, reset, DNS)
36
+ - Server errors (5xx) -- includes S3 503 SlowDown
37
+ Not retryable:
38
+ - Client errors (4xx) -- bad URL, auth, not-found
39
+ - Programming errors (ValueError, TypeError, ...)
40
+ """
41
+ # Timeouts
42
+ if isinstance(
43
+ exc,
44
+ (
45
+ TimeoutError,
46
+ asyncio.TimeoutError,
47
+ requests.exceptions.Timeout,
48
+ aiohttp.ServerTimeoutError,
49
+ ),
50
+ ):
51
+ return True
52
+ # Connection-level failures
53
+ if isinstance(
54
+ exc,
55
+ (
56
+ ConnectionError,
57
+ aiohttp.ClientConnectionError,
58
+ requests.exceptions.ConnectionError,
59
+ ),
60
+ ):
61
+ return True
62
+ # aiohttp server-side disconnects
63
+ if isinstance(exc, aiohttp.ServerDisconnectedError):
64
+ return True
65
+ # requests 5xx -- raise_for_status() throws HTTPError
66
+ if (
67
+ isinstance(exc, requests.exceptions.HTTPError)
68
+ and exc.response is not None
69
+ and exc.response.status_code >= 500
70
+ ):
71
+ return True
72
+ # aiohttp 5xx -- raise_for_status() throws ClientResponseError
73
+ return isinstance(exc, aiohttp.ClientResponseError) and exc.status >= 500
74
+
75
+
76
+def _log_retry(
77
+ args: tuple,
78
+ kwargs: dict,
79
+ attempt: int,
80
+ max_retries: int,
81
+ attempt_timeout: float | None,
82
+ exc: Exception,
83
+ backoff: float,
84
+ base_timeout: float | None,
85
+) -> None:
86
+ # args[0] is `self` (bound method), args[1] is the URL
87
+ url = args[1] if len(args) > 1 else kwargs.get("url")
88
+ timeout_info = (
89
+ f"timeout={attempt_timeout:.3f}s" if base_timeout is not None else "no timeout"
90
+ )
91
+ next_timeout = (
92
+ f" with timeout={base_timeout * (_RETRY_BACKOFF_FACTOR ** (attempt + 1)):.3f}s"
93
+ if base_timeout is not None
94
+ else ""
95
+ )
96
+ logger.warning(
97
+ "HTTP fetch failed for %s (attempt %d/%d, %s): %s -- retrying in %.3fs%s",
98
+ url,
99
+ attempt + 1,
100
+ max_retries,
101
+ timeout_info,
102
+ exc,
103
+ backoff,
104
+ next_timeout,
105
+ )
106
+
107
+
108
+def _sync_retry(
109
+ fn: Callable[_P, _T],
110
+) -> Callable[_P, _T]:
111
+ """Add retry logic with exponential backoff to a sync method.
112
+
113
+ The decorated method must accept ``timeout`` as a keyword argument.
114
+ The decorator replaces it with a per-attempt timeout that grows by
115
+ ``_RETRY_BACKOFF_FACTOR`` on each retry so transient slowness on busy
116
+ hosts is absorbed.
117
+ """
118
+
119
+ @functools.wraps(fn)
120
+ def wrapper(*args: Any, **kwargs: Any) -> _T:
121
+ base_timeout: float | None = kwargs.get("timeout")
122
+ max_retries = max(envs.VLLM_MEDIA_FETCH_MAX_RETRIES, 1)
123
+
124
+ for attempt in range(max_retries):
125
+ attempt_timeout = (
126
+ base_timeout * (_RETRY_BACKOFF_FACTOR**attempt)
127
+ if base_timeout is not None
128
+ else None
129
+ )
130
+ kwargs["timeout"] = attempt_timeout
131
+ try:
132
+ return fn(*args, **kwargs)
133
+ except Exception as e:
134
+ if not _is_retryable(e) or attempt + 1 >= max_retries:
135
+ raise
136
+ backoff = _RETRY_BACKOFF_FACTOR**attempt
137
+ _log_retry(
138
+ args,
139
+ kwargs,
140
+ attempt,
141
+ max_retries,
142
+ attempt_timeout,
143
+ e,
144
+ backoff,
145
+ base_timeout,
146
+ )
147
+ time.sleep(backoff)
148
+
149
+ raise AssertionError("unreachable")
150
+
151
+ return wrapper # type: ignore[return-value]
152
+
153
+
154
+def _async_retry(
155
+ fn: Callable[_P, Coroutine[Any, Any, _T]],
156
+) -> Callable[_P, Coroutine[Any, Any, _T]]:
157
+ """Add retry logic with exponential backoff to an async method.
158
+
159
+ The decorated method must accept ``timeout`` as a keyword argument.
160
+ The decorator replaces it with a per-attempt timeout that grows by
161
+ ``_RETRY_BACKOFF_FACTOR`` on each retry so transient slowness on busy
162
+ hosts is absorbed.
163
+ """
164
+
165
+ @functools.wraps(fn)
166
+ async def wrapper(*args: Any, **kwargs: Any) -> _T:
167
+ base_timeout: float | None = kwargs.get("timeout")
168
+ max_retries = max(envs.VLLM_MEDIA_FETCH_MAX_RETRIES, 1)
169
+
170
+ for attempt in range(max_retries):
171
+ attempt_timeout = (
172
+ base_timeout * (_RETRY_BACKOFF_FACTOR**attempt)
173
+ if base_timeout is not None
174
+ else None
175
+ )
176
+ kwargs["timeout"] = attempt_timeout
177
+ try:
178
+ return await fn(*args, **kwargs)
179
+ except Exception as e:
180
+ if not _is_retryable(e) or attempt + 1 >= max_retries:
181
+ raise
182
+ backoff = _RETRY_BACKOFF_FACTOR**attempt
183
+ _log_retry(
184
+ args,
185
+ kwargs,
186
+ attempt,
187
+ max_retries,
188
+ attempt_timeout,
189
+ e,
190
+ backoff,
191
+ base_timeout,
192
+ )
193
+ await asyncio.sleep(backoff)
194
+
195
+ raise AssertionError("unreachable")
196
+
197
+ return wrapper # type: ignore[return-value]
198
+
199
200
class HTTPConnection:
201
"""Helper class to send HTTP requests."""
@@ -89,6 +275,7 @@ class HTTPConnection:
275
allow_redirects=allow_redirects,
276
)
277
278
+ @_sync_retry
279
def get_bytes(
280
self, url: str, *, timeout: float | None = None, allow_redirects: bool = True
281
) -> bytes:
@@ -99,6 +286,7 @@ class HTTPConnection:
286
287
return r.content
288
289
+ @_async_retry
290
async def async_get_bytes(
291
self,
292
url: str,
@@ -147,6 +335,7 @@ class HTTPConnection:
335
336
return await r.json()
337
338
+ @_sync_retry
339
def download_file(
340
self,
341
url: str,
@@ -155,15 +344,22 @@ class HTTPConnection:
344
timeout: float | None = None,
345
chunk_size: int = 128,
346
) -> Path:
158
- with self.get_response(url, timeout=timeout) as r:
159
- r.raise_for_status()
160
-
161
- with save_path.open("wb") as f:
162
- for chunk in r.iter_content(chunk_size):
163
- f.write(chunk)
164
-
165
- return save_path
166
-
347
+ try:
348
+ with self.get_response(url, timeout=timeout) as r:
349
+ r.raise_for_status()
350
+
351
+ with save_path.open("wb") as f:
352
+ for chunk in r.iter_content(chunk_size):
353
+ f.write(chunk)
354
+
355
+ return save_path
356
+ except Exception:
357
+ # Clean up partial downloads before retrying or propagating
358
+ if save_path.exists():
359
+ save_path.unlink()
360
+ raise
361
+
362
+ @_async_retry
363
async def async_download_file(
364
self,
365
url: str,
@@ -172,14 +368,23 @@ class HTTPConnection:
368
timeout: float | None = None,
369
chunk_size: int = 128,
370
) -> Path:
175
- async with await self.get_async_response(url, timeout=timeout) as r:
176
- r.raise_for_status()
177
-
178
- with save_path.open("wb") as f:
179
- async for chunk in r.content.iter_chunked(chunk_size):
180
- f.write(chunk)
181
-
182
- return save_path
371
+ try:
372
+ async with await self.get_async_response(
373
+ url,
374
+ timeout=timeout,
375
+ ) as r:
376
+ r.raise_for_status()
377
+
378
+ with save_path.open("wb") as f:
379
+ async for chunk in r.content.iter_chunked(chunk_size):
380
+ f.write(chunk)
381
+
382
+ return save_path
383
+ except Exception:
384
+ # Clean up partial downloads before retrying or propagating
385
+ if save_path.exists():
386
+ save_path.unlink()
387
+ raise
388
389
390
global_http_connection = HTTPConnection()
vllm/envs.py
+7
@@ -64,6 +64,7 @@ if TYPE_CHECKING:
64
VLLM_IMAGE_FETCH_TIMEOUT: int = 5
65
VLLM_VIDEO_FETCH_TIMEOUT: int = 30
66
VLLM_AUDIO_FETCH_TIMEOUT: int = 10
67
+ VLLM_MEDIA_FETCH_MAX_RETRIES: int = 3
68
VLLM_MEDIA_URL_ALLOW_REDIRECTS: bool = True
69
VLLM_MEDIA_LOADING_THREAD_COUNT: int = 8
70
VLLM_MAX_AUDIO_CLIP_FILESIZE_MB: int = 25
@@ -773,6 +774,11 @@ environment_variables: dict[str, Callable[[], Any]] = {
774
"VLLM_AUDIO_FETCH_TIMEOUT": lambda: int(
775
os.getenv("VLLM_AUDIO_FETCH_TIMEOUT", "10")
776
),
777
+ # Maximum number of retries for fetching media (images, audio, video)
778
+ # from URLs. Each retry quadruples the timeout. Default is 3.
779
+ "VLLM_MEDIA_FETCH_MAX_RETRIES": lambda: int(
780
+ os.getenv("VLLM_MEDIA_FETCH_MAX_RETRIES", "3")
781
+ ),
782
# Whether to allow HTTP redirects when fetching from media URLs.
783
# Default to True
784
"VLLM_MEDIA_URL_ALLOW_REDIRECTS": lambda: bool(
@@ -1768,6 +1774,7 @@ def compile_factors() -> dict[str, object]:
1774
"VLLM_IMAGE_FETCH_TIMEOUT",
1775
"VLLM_VIDEO_FETCH_TIMEOUT",
1776
"VLLM_AUDIO_FETCH_TIMEOUT",
1777
+ "VLLM_MEDIA_FETCH_MAX_RETRIES",
1778
"VLLM_MEDIA_URL_ALLOW_REDIRECTS",
1779
"VLLM_MEDIA_LOADING_THREAD_COUNT",
1780
"VLLM_MAX_AUDIO_CLIP_FILESIZE_MB",