@tduncan / Nvidia-Optimization-Stack-VLLM / commits / 2ec6f0d71e

Reject oversized media before fully downloading it (#51896)

Signed-off-by: Clinton Thomas <1033162+KernelClint@users.noreply.github.com> Co-authored-by: Lucas Bourtoule <35483370+dhalf@users.noreply.github.com>

Clinton Thomas committed Aug 23, 2026 at 21:15 UTC 2ec6f0d71ea3b350952630e310efcda1c744ff4d
13 files changed +718 -24
docs/usage/security.md
+5 -3
@@ -85,13 +85,15 @@ significantly reduce the attack surface for these types of abuse.
85 Also, consider setting `VLLM_MEDIA_URL_ALLOW_REDIRECTS=0` to prevent HTTP
86 redirects from being followed to bypass domain restrictions.
87
88 -### 5. **Restrict Media Decode Sizes:**
88 +### 5. **Restrict Media Download and Decode Sizes:**
89
90 -Compressed media files can expand into gigabytes of memory during decoding. vLLM
91 -enforces decode-size limits to prevent out-of-memory denial of service:
90 +Remote media responses and compressed media files can expand into gigabytes of
91 +memory. vLLM enforces download and decode-size limits to prevent out-of-memory
92 +denial of service:
93
94 | Environment Variable | Default | Description |
95 | --- | --- | --- |
96 +| `VLLM_MAX_MEDIA_DOWNLOAD_SIZE_MB` | `256` | Maximum size in MB for a single remote media response. Oversized responses are rejected while streaming before the full body is materialized in memory. |
97 | `VLLM_MAX_IMAGE_PIXELS` | `178956970` (~179M pixels) | Maximum decoded image size in pixels. Images exceeding this are rejected before raster memory is allocated. Default matches PIL's built-in 2x decompression-bomb threshold (~680 MB for RGB). |
98 | `VLLM_MAX_AUDIO_CLIP_FILESIZE_MB` | `25` | Maximum filesize in MB for a single audio file. |
99 | `VLLM_MAX_AUDIO_DECODE_DURATION_S` | `600` | Maximum decoded audio duration in seconds. Prevents compressed audio from expanding into gigabytes of float32 PCM. |
tests/entrypoints/openai/test_run_batch.py
+160
@@ -4,16 +4,25 @@
4 import json
5 import subprocess
6 import tempfile
7 +import threading
8 +from collections.abc import Generator
9 +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
10 from unittest.mock import AsyncMock, MagicMock, patch
11
12 import pytest
13
14 +import vllm.envs as envs
15 from vllm.assets.audio import AudioAsset
16 +from vllm.connections import HTTPConnection
17 +from vllm.entrypoints.openai.engine.protocol import ErrorResponse
18 from vllm.entrypoints.openai.run_batch import (
19 BatchRequestOutput,
20 + BatchTranscriptionRequest,
21 download_bytes_from_url,
22 + make_transcription_wrapper,
23 )
24 from vllm.exceptions import VLLMValidationError
25 +from vllm.utils.mem_constants import MiB_bytes
26
27 CHAT_MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
28 EMBEDDING_MODEL_NAME = "intfloat/multilingual-e5-small"
@@ -281,6 +290,74 @@ INPUT_REASONING_BATCH = "\n".join(
290 )
291
292 MINIMAL_WAV_BASE64 = "UklGRigAAABXQVZFZm10IBAAAAABAAEAgD4AAAB9AAACABAAZGF0YQQAAAAAAP9/"
293 +_EXACT_LIMIT_AUDIO = b"a" * MiB_bytes
294 +_OVERSIZED_BASE64_AUDIO = "A" * (4 * ((MiB_bytes + 2) // 3))
295 +_OVERSIZED_HTTP_AUDIO = b"b" * (MiB_bytes + 1)
296 +
297 +
298 +class _BatchHTTPServer(ThreadingHTTPServer):
299 + request_paths: list[str]
300 +
301 +
302 +class _BatchHTTPHandler(BaseHTTPRequestHandler):
303 + def do_GET(self) -> None:
304 + server = self.server
305 + assert isinstance(server, _BatchHTTPServer)
306 + server.request_paths.append(self.path)
307 +
308 + if self.path == "/exact-limit":
309 + self._send_body(_EXACT_LIMIT_AUDIO)
310 + return
311 +
312 + if self.path == "/oversized-chunked":
313 + self.send_response(200)
314 + self.end_headers()
315 + self._write_body(_OVERSIZED_HTTP_AUDIO)
316 + return
317 +
318 + self.send_error(404, "Unknown batch HTTP test path")
319 +
320 + def _send_body(self, body: bytes) -> None:
321 + self.send_response(200)
322 + self.send_header("Content-Type", "application/octet-stream")
323 + self.send_header("Content-Length", str(len(body)))
324 + self.end_headers()
325 + self._write_body(body)
326 +
327 + def _write_body(self, body: bytes) -> None:
328 + try:
329 + for offset in range(0, len(body), 64 * 1024):
330 + self.wfile.write(body[offset : offset + 64 * 1024])
331 + except (BrokenPipeError, ConnectionResetError):
332 + self.close_connection = True
333 +
334 + def log_message(self, fmt: str, *args: object) -> None:
335 + return
336 +
337 +
338 +@pytest.fixture
339 +def batch_http_server() -> Generator[_BatchHTTPServer, None, None]:
340 + server = _BatchHTTPServer(("127.0.0.1", 0), _BatchHTTPHandler)
341 + server.request_paths = []
342 + thread = threading.Thread(target=server.serve_forever, daemon=True)
343 + thread.start()
344 + try:
345 + yield server
346 + finally:
347 + server.shutdown()
348 + thread.join()
349 + server.server_close()
350 +
351 +
352 +def _batch_http_url(server: _BatchHTTPServer, path: str) -> str:
353 + return f"http://localhost:{server.server_port}{path}"
354 +
355 +
356 +async def _close_async_connection(connection: HTTPConnection) -> None:
357 + if connection._async_client is not None:
358 + await connection._async_client.close()
359 +
360 +
361 INPUT_TRANSCRIPTION_BATCH = (
362 json.dumps(
363 {
@@ -761,8 +838,15 @@ def test_tool_calling():
838
839 def _make_aiohttp_mocks(response_data: bytes = b"fake-data", status: int = 200):
840 """Create mock objects that simulate aiohttp.ClientSession context managers."""
841 +
842 + async def iter_chunked(chunk_size: int):
843 + del chunk_size
844 + yield response_data
845 +
846 mock_resp = MagicMock()
847 mock_resp.status = status
848 + mock_resp.content_length = len(response_data)
849 + mock_resp.content.iter_chunked = iter_chunked
850 mock_resp.read = AsyncMock(return_value=response_data)
851 mock_resp.__aenter__ = AsyncMock(return_value=mock_resp)
852 mock_resp.__aexit__ = AsyncMock(return_value=False)
@@ -892,3 +976,79 @@ async def test_download_bytes_backslash_bypass():
976 await download_bytes_from_url(
977 bypass_url, allowed_media_domains=["evil.internal"]
978 )
979 +
980 +
981 +@pytest.mark.asyncio
982 +async def test_transcription_wrapper_rejects_oversized_data_url_before_decode(
983 + monkeypatch: pytest.MonkeyPatch,
984 +):
985 + monkeypatch.setattr(envs, "VLLM_MAX_AUDIO_CLIP_FILESIZE_MB", 1)
986 + handler = AsyncMock()
987 + wrapped_handler = make_transcription_wrapper(is_translation=False)(handler)
988 + request = BatchTranscriptionRequest.model_validate(
989 + {
990 + "model": SPEECH_LARGE_MODEL_NAME,
991 + "file_url": f"data:audio/wav;base64,{_OVERSIZED_BASE64_AUDIO}",
992 + "response_format": "json",
993 + }
994 + )
995 +
996 + with patch("vllm.entrypoints.openai.run_batch.base64.b64decode") as decode:
997 + response = await wrapped_handler(request)
998 +
999 + assert isinstance(response, ErrorResponse)
1000 + assert "Maximum file size exceeded" in response.error.message
1001 + decode.assert_not_called()
1002 + handler.assert_not_awaited()
1003 +
1004 +
1005 +@pytest.mark.asyncio
1006 +async def test_download_bytes_allows_http_body_at_audio_limit(
1007 + monkeypatch: pytest.MonkeyPatch,
1008 + batch_http_server: _BatchHTTPServer,
1009 +):
1010 + monkeypatch.setattr(envs, "VLLM_MAX_AUDIO_CLIP_FILESIZE_MB", 1)
1011 + connection = HTTPConnection()
1012 + monkeypatch.setattr(
1013 + "vllm.entrypoints.openai.run_batch.global_http_connection",
1014 + connection,
1015 + )
1016 +
1017 + try:
1018 + result = await download_bytes_from_url(
1019 + _batch_http_url(batch_http_server, "/exact-limit")
1020 + )
1021 + assert result == _EXACT_LIMIT_AUDIO
1022 + finally:
1023 + await _close_async_connection(connection)
1024 +
1025 +
1026 +@pytest.mark.asyncio
1027 +async def test_transcription_wrapper_rejects_oversized_http_before_handler(
1028 + monkeypatch: pytest.MonkeyPatch,
1029 + batch_http_server: _BatchHTTPServer,
1030 +):
1031 + monkeypatch.setattr(envs, "VLLM_MAX_AUDIO_CLIP_FILESIZE_MB", 1)
1032 + connection = HTTPConnection()
1033 + monkeypatch.setattr(
1034 + "vllm.entrypoints.openai.run_batch.global_http_connection",
1035 + connection,
1036 + )
1037 + handler = AsyncMock()
1038 + wrapped_handler = make_transcription_wrapper(is_translation=False)(handler)
1039 + request = BatchTranscriptionRequest.model_validate(
1040 + {
1041 + "model": SPEECH_LARGE_MODEL_NAME,
1042 + "file_url": _batch_http_url(batch_http_server, "/oversized-chunked"),
1043 + "response_format": "json",
1044 + }
1045 + )
1046 +
1047 + try:
1048 + response = await wrapped_handler(request)
1049 + finally:
1050 + await _close_async_connection(connection)
1051 +
1052 + assert isinstance(response, ErrorResponse)
1053 + assert "Maximum file size exceeded" in response.error.message
1054 + handler.assert_not_awaited()
tests/entrypoints/speech_to_text/transcription/test_chunk_timestamp_offset.py
-1
@@ -41,7 +41,6 @@ async def test_chunk_offsets_are_cumulative_not_nominal():
41 overlap_chunk_second=1,
42 min_energy_split_window_size=1600,
43 )
44 - serving.max_audio_filesize_mb = 100.0
44 serving.model_cls = MagicMock()
45 serving.model_cls.validate_language.side_effect = lambda lang: lang
46 serving.model_cls.supports_explicit_language_detection = False
tests/multimodal/media/test_unprocessable_entity_error.py
+14
@@ -13,6 +13,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
13 import aiohttp
14 import pytest
15
16 +from vllm.connections import MediaDownloadSizeExceededError
17 from vllm.entrypoints.serve import create_error_response
18 from vllm.exceptions import VLLMClientError, VLLMUnprocessableEntityError
19 from vllm.multimodal.media import MediaConnector
@@ -134,6 +135,19 @@ class TestMediaConnectorErrorHandling:
135
136 assert isinstance(exc_info.value, aiohttp.ClientConnectionError)
137
138 + def test_fetch_image_download_size_limit_error(self):
139 + connector = MediaConnector()
140 +
141 + with patch.object(
142 + connector.connection, "get_bytes", new_callable=MagicMock
143 + ) as mock_get:
144 + mock_get.side_effect = MediaDownloadSizeExceededError(256 * 1024 * 1024)
145 +
146 + with pytest.raises(VLLMUnprocessableEntityError) as exc_info:
147 + connector.fetch_image("https://example.com/image.jpg")
148 +
149 + assert "VLLM_MAX_MEDIA_DOWNLOAD_SIZE_MB=256" in str(exc_info.value)
150 +
151
152 class TestErrorResponse:
153 """Tests for error response creation."""
tests/test_audio_media_size_precheck.py new
+129
@@ -0,0 +1,129 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +from collections.abc import AsyncIterator, Iterator
5 +from unittest.mock import patch
6 +
7 +import pytest
8 +
9 +import vllm.envs as envs
10 +from vllm.connections import HTTPConnection
11 +from vllm.exceptions import VLLMValidationError
12 +from vllm.multimodal.media import AudioMediaIO, MediaConnector
13 +from vllm.utils.mem_constants import KiB_bytes, MiB_bytes
14 +
15 +pytestmark = [pytest.mark.cpu_test, pytest.mark.skip_global_cleanup]
16 +
17 +
18 +class _SyncResponse:
19 + def __init__(self, chunks: list[bytes], content_length: int | None = None):
20 + self.headers = (
21 + {} if content_length is None else {"content-length": str(content_length)}
22 + )
23 + self._chunks = chunks
24 + self.iterated = 0
25 +
26 + def __enter__(self):
27 + return self
28 +
29 + def __exit__(self, *_args):
30 + return None
31 +
32 + def raise_for_status(self) -> None:
33 + return None
34 +
35 + def iter_content(self, chunk_size: int) -> Iterator[bytes]:
36 + assert chunk_size == 64 * KiB_bytes
37 + for chunk in self._chunks:
38 + self.iterated += 1
39 + yield chunk
40 +
41 +
42 +class _AsyncContent:
43 + def __init__(self, chunks: list[bytes]):
44 + self._chunks = chunks
45 + self.iterated = 0
46 +
47 + async def iter_chunked(self, chunk_size: int) -> AsyncIterator[bytes]:
48 + assert chunk_size == 64 * KiB_bytes
49 + for chunk in self._chunks:
50 + self.iterated += 1
51 + yield chunk
52 +
53 +
54 +class _AsyncResponse:
55 + def __init__(self, chunks: list[bytes], content_length: int | None = None):
56 + self.content_length = content_length
57 + self.content = _AsyncContent(chunks)
58 +
59 + async def __aenter__(self):
60 + return self
61 +
62 + async def __aexit__(self, *_args):
63 + return None
64 +
65 + def raise_for_status(self) -> None:
66 + return None
67 +
68 +
69 +def test_audio_base64_rejects_before_decode(monkeypatch: pytest.MonkeyPatch):
70 + monkeypatch.setattr(envs, "VLLM_MAX_AUDIO_CLIP_FILESIZE_MB", 1)
71 + max_encoded_chars = 4 * ((MiB_bytes + 2) // 3)
72 +
73 + with (
74 + patch("vllm.multimodal.media.audio.pybase64.b64decode") as decode,
75 + pytest.raises(VLLMValidationError, match="Maximum file size exceeded"),
76 + ):
77 + AudioMediaIO().load_base64("audio/wav", "A" * (max_encoded_chars + 1))
78 +
79 + decode.assert_not_called()
80 +
81 +
82 +def test_sync_http_reader_rejects_from_content_length_before_body_read():
83 + response = _SyncResponse([b"A" * 10], content_length=MiB_bytes + 1)
84 + connection = HTTPConnection()
85 +
86 + with (
87 + patch.object(connection, "get_response", return_value=response) as get_response,
88 + pytest.raises(VLLMValidationError, match="Maximum file size exceeded"),
89 + ):
90 + connection.get_bytes("https://example.test/audio", max_bytes=MiB_bytes)
91 +
92 + assert get_response.call_args.kwargs["stream"] is True
93 + assert response.iterated == 0
94 +
95 +
96 +@pytest.mark.asyncio
97 +async def test_async_http_reader_stops_after_first_over_limit_chunk():
98 + response = _AsyncResponse([b"A" * (64 * KiB_bytes), b"B", b"C" * 10])
99 + connection = HTTPConnection()
100 +
101 + async def get_async_response(*_args, **_kwargs):
102 + return response
103 +
104 + with (
105 + patch.object(connection, "get_async_response", side_effect=get_async_response),
106 + pytest.raises(VLLMValidationError, match="Maximum file size exceeded"),
107 + ):
108 + await connection.async_get_bytes(
109 + "https://example.test/audio",
110 + max_bytes=64 * KiB_bytes,
111 + )
112 +
113 + assert response.content.iterated == 2
114 +
115 +
116 +@pytest.mark.asyncio
117 +async def test_audio_connector_threads_byte_limit_to_http_reader(
118 + monkeypatch: pytest.MonkeyPatch,
119 +):
120 + monkeypatch.setattr(envs, "VLLM_MAX_AUDIO_CLIP_FILESIZE_MB", 1)
121 +
122 + class _Connection:
123 + async def async_get_bytes(self, _url: str, **kwargs):
124 + assert kwargs["max_bytes"] == MiB_bytes
125 + raise VLLMValidationError("Maximum file size exceeded")
126 +
127 + connector = MediaConnector(connection=_Connection()) # type: ignore[arg-type]
128 + with pytest.raises(VLLMValidationError, match="Maximum file size exceeded"):
129 + await connector.fetch_audio_async("https://example.test/audio")
tests/test_connections.py new
+192
@@ -0,0 +1,192 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +import threading
5 +from collections.abc import Generator
6 +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
7 +from unittest.mock import AsyncMock, patch
8 +
9 +import pytest
10 +
11 +import vllm.envs as envs
12 +from vllm.connections import HTTPConnection, HTTPResponseSizeExceededError
13 +
14 +_ONE_MIB = 1024 * 1024
15 +_SMALL_BODY = b"a" * (_ONE_MIB // 2)
16 +_LARGE_BODY = b"b" * (2 * _ONE_MIB)
17 +_CHUNK_SIZE = 64 * 1024
18 +
19 +
20 +class _MediaHandler(BaseHTTPRequestHandler):
21 + def do_GET(self) -> None:
22 + body = _LARGE_BODY if "large" in self.path else _SMALL_BODY
23 + self.send_response(200)
24 + self.send_header("Content-Type", "application/octet-stream")
25 + if self.path != "/chunked-large":
26 + self.send_header("Content-Length", str(len(body)))
27 + self.end_headers()
28 +
29 + try:
30 + for offset in range(0, len(body), _CHUNK_SIZE):
31 + self.wfile.write(body[offset : offset + _CHUNK_SIZE])
32 + except (BrokenPipeError, ConnectionResetError):
33 + self.close_connection = True
34 +
35 + def log_message(self, fmt: str, *args: object) -> None:
36 + return
37 +
38 +
39 +@pytest.fixture
40 +def local_media_server() -> Generator[str, None, None]:
41 + server = ThreadingHTTPServer(("127.0.0.1", 0), _MediaHandler)
42 + thread = threading.Thread(target=server.serve_forever, daemon=True)
43 + thread.start()
44 + try:
45 + yield f"http://127.0.0.1:{server.server_port}"
46 + finally:
47 + server.shutdown()
48 + thread.join()
49 + server.server_close()
50 +
51 +
52 +class _SyncResponse:
53 + def __init__(self, chunks: list[bytes]) -> None:
54 + self.headers: dict[str, str] = {}
55 + self._chunks = chunks
56 + self.content_accessed = False
57 + self.iterated = 0
58 +
59 + @property
60 + def content(self) -> bytes:
61 + self.content_accessed = True
62 + raise AssertionError("bounded reads must not access response.content")
63 +
64 + def iter_content(self, chunk_size: int):
65 + del chunk_size
66 + for chunk in self._chunks:
67 + self.iterated += 1
68 + yield chunk
69 +
70 + def raise_for_status(self) -> None:
71 + return
72 +
73 + def __enter__(self):
74 + return self
75 +
76 + def __exit__(self, exc_type, exc_value, traceback):
77 + return False
78 +
79 +
80 +class _AsyncContent:
81 + def __init__(self, chunks: list[bytes]) -> None:
82 + self._chunks = chunks
83 + self.iterated = 0
84 +
85 + async def iter_chunked(self, chunk_size: int):
86 + del chunk_size
87 + for chunk in self._chunks:
88 + self.iterated += 1
89 + yield chunk
90 +
91 +
92 +class _AsyncResponse:
93 + def __init__(self, chunks: list[bytes]) -> None:
94 + self.content = _AsyncContent(chunks)
95 + self.content_length = None
96 + self.read = AsyncMock(
97 + side_effect=AssertionError("bounded reads must not call response.read()")
98 + )
99 +
100 + def raise_for_status(self) -> None:
101 + return
102 +
103 + async def __aenter__(self):
104 + return self
105 +
106 + async def __aexit__(self, exc_type, exc_value, traceback):
107 + return False
108 +
109 +
110 +def test_get_bytes_with_limit_streams_instead_of_materializing() -> None:
111 + response = _SyncResponse([b"abcd", b"efgh", b"ijkl"])
112 + connection = HTTPConnection()
113 +
114 + with (
115 + patch.object(
116 + connection,
117 + "get_response",
118 + return_value=response,
119 + ) as get_response,
120 + pytest.raises(HTTPResponseSizeExceededError, match="maximum size"),
121 + ):
122 + connection.get_bytes("http://example.com/media", max_bytes=4)
123 +
124 + get_response.assert_called_once_with(
125 + "http://example.com/media",
126 + stream=True,
127 + timeout=None,
128 + allow_redirects=True,
129 + )
130 + assert response.content_accessed is False
131 + assert response.iterated == 2
132 +
133 +
134 +@pytest.mark.asyncio
135 +async def test_async_get_bytes_with_limit_streams_instead_of_materializing() -> None:
136 + response = _AsyncResponse([b"abcd", b"efgh", b"ijkl"])
137 + connection = HTTPConnection()
138 +
139 + with (
140 + patch.object(
141 + connection,
142 + "get_async_response",
143 + new=AsyncMock(return_value=response),
144 + ),
145 + pytest.raises(HTTPResponseSizeExceededError, match="maximum size"),
146 + ):
147 + await connection.async_get_bytes("http://example.com/media", max_bytes=4)
148 +
149 + response.read.assert_not_awaited()
150 + assert response.content.iterated == 2
151 +
152 +
153 +def test_get_bytes_rejects_content_length_over_limit(
154 + monkeypatch: pytest.MonkeyPatch, local_media_server: str
155 +) -> None:
156 + monkeypatch.setattr(envs, "VLLM_MAX_MEDIA_DOWNLOAD_SIZE_MB", 1, raising=False)
157 + connection = HTTPConnection(reuse_client=False)
158 +
159 + with pytest.raises(ValueError, match="VLLM_MAX_MEDIA_DOWNLOAD_SIZE_MB"):
160 + connection.get_bytes(f"{local_media_server}/large")
161 +
162 +
163 +@pytest.mark.asyncio
164 +async def test_async_get_bytes_rejects_stream_over_limit_without_content_length(
165 + monkeypatch: pytest.MonkeyPatch, local_media_server: str
166 +) -> None:
167 + monkeypatch.setattr(envs, "VLLM_MAX_MEDIA_DOWNLOAD_SIZE_MB", 1, raising=False)
168 + connection = HTTPConnection(reuse_client=False)
169 +
170 + try:
171 + with pytest.raises(ValueError, match="VLLM_MAX_MEDIA_DOWNLOAD_SIZE_MB"):
172 + await connection.async_get_bytes(f"{local_media_server}/chunked-large")
173 + finally:
174 + if connection._async_client is not None:
175 + await connection._async_client.close()
176 +
177 +
178 +@pytest.mark.asyncio
179 +async def test_get_bytes_allows_body_within_limit(
180 + monkeypatch: pytest.MonkeyPatch, local_media_server: str
181 +) -> None:
182 + monkeypatch.setattr(envs, "VLLM_MAX_MEDIA_DOWNLOAD_SIZE_MB", 1, raising=False)
183 + connection = HTTPConnection(reuse_client=False)
184 +
185 + assert connection.get_bytes(f"{local_media_server}/small") == _SMALL_BODY
186 +
187 + try:
188 + actual = await connection.async_get_bytes(f"{local_media_server}/small")
189 + assert actual == _SMALL_BODY
190 + finally:
191 + if connection._async_client is not None:
192 + await connection._async_client.close()
vllm/connections.py
+140 -5
@@ -13,7 +13,9 @@ import requests
13 from urllib3.util import parse_url
14
15 import vllm.envs as envs
16 +from vllm.exceptions import VLLMValidationError
17 from vllm.logger import init_logger
18 +from vllm.utils.mem_constants import KiB_bytes, MiB_bytes
19 from vllm.version import __version__ as VLLM_VERSION
20
21 logger = init_logger(__name__)
@@ -25,6 +27,128 @@ _T = TypeVar("_T")
27 # Attempt N uses: base_timeout * (_RETRY_BACKOFF_FACTOR ** N) for the
28 # per-attempt timeout and sleeps _RETRY_BACKOFF_FACTOR ** N seconds.
29 _RETRY_BACKOFF_FACTOR = 4
30 +_RESPONSE_READ_CHUNK_SIZE = 64 * KiB_bytes
31 +
32 +
33 +class HTTPResponseSizeExceededError(VLLMValidationError):
34 + """Raised when an HTTP response exceeds a caller-supplied byte ceiling."""
35 +
36 + def __init__(self, max_bytes: int, received_bytes: int) -> None:
37 + self.max_bytes = max_bytes
38 + self.received_bytes = received_bytes
39 + super().__init__(
40 + "Maximum file size exceeded: HTTP response exceeded maximum size "
41 + f"of {max_bytes} bytes (received at least {received_bytes} bytes)",
42 + parameter="audio_filesize_mb",
43 + value=received_bytes / MiB_bytes,
44 + )
45 +
46 +
47 +class MediaDownloadSizeExceededError(ValueError):
48 + """Raised when a remote media download exceeds the configured byte limit."""
49 +
50 + def __init__(self, max_bytes: int, received_bytes: int | None = None) -> None:
51 + self.max_bytes = max_bytes
52 + self.received_bytes = received_bytes
53 + max_size_mb = max_bytes / MiB_bytes
54 + message = (
55 + "Remote media download exceeds "
56 + f"VLLM_MAX_MEDIA_DOWNLOAD_SIZE_MB={max_size_mb:g}"
57 + )
58 + if received_bytes is not None:
59 + message += f" (received at least {received_bytes} bytes)"
60 + super().__init__(message)
61 +
62 +
63 +def _get_media_download_limit_bytes() -> int | None:
64 + max_size_mb = envs.VLLM_MAX_MEDIA_DOWNLOAD_SIZE_MB
65 + if max_size_mb <= 0:
66 + return None
67 + return max_size_mb * MiB_bytes
68 +
69 +
70 +def _get_response_size_limit(max_bytes: int | None) -> tuple[int | None, bool]:
71 + media_max_bytes = _get_media_download_limit_bytes()
72 + if media_max_bytes is not None and (
73 + max_bytes is None or media_max_bytes <= max_bytes
74 + ):
75 + return media_max_bytes, True
76 + return max_bytes, False
77 +
78 +
79 +def _raise_size_exceeded(
80 + max_bytes: int, received_bytes: int, media_download_limit: bool
81 +) -> None:
82 + if media_download_limit:
83 + raise MediaDownloadSizeExceededError(max_bytes, received_bytes)
84 + raise HTTPResponseSizeExceededError(max_bytes, received_bytes)
85 +
86 +
87 +def _check_content_length(
88 + content_length: str | int | None,
89 + max_bytes: int | None,
90 + media_download_limit: bool,
91 +) -> None:
92 + if content_length is None or max_bytes is None:
93 + return
94 + try:
95 + parsed_length = int(content_length)
96 + except (TypeError, ValueError):
97 + return
98 + if parsed_length > max_bytes:
99 + _raise_size_exceeded(max_bytes, parsed_length, media_download_limit)
100 +
101 +
102 +def _append_chunk(
103 + chunks: list[bytes],
104 + chunk: bytes,
105 + received_bytes: int,
106 + max_bytes: int | None,
107 + media_download_limit: bool,
108 +) -> int:
109 + if not chunk:
110 + return received_bytes
111 + received_bytes += len(chunk)
112 + if max_bytes is not None and received_bytes > max_bytes:
113 + _raise_size_exceeded(max_bytes, received_bytes, media_download_limit)
114 + chunks.append(chunk)
115 + return received_bytes
116 +
117 +
118 +def _read_response_bytes(response: requests.Response, max_bytes: int | None) -> bytes:
119 + max_bytes, media_download_limit = _get_response_size_limit(max_bytes)
120 + if max_bytes is None:
121 + return response.content
122 +
123 + _check_content_length(
124 + response.headers.get("Content-Length"),
125 + max_bytes,
126 + media_download_limit,
127 + )
128 + chunks: list[bytes] = []
129 + received_bytes = 0
130 + for chunk in response.iter_content(_RESPONSE_READ_CHUNK_SIZE):
131 + received_bytes = _append_chunk(
132 + chunks, chunk, received_bytes, max_bytes, media_download_limit
133 + )
134 + return b"".join(chunks)
135 +
136 +
137 +async def _async_read_response_bytes(
138 + response: aiohttp.ClientResponse, max_bytes: int | None
139 +) -> bytes:
140 + max_bytes, media_download_limit = _get_response_size_limit(max_bytes)
141 + if max_bytes is None:
142 + return await response.read()
143 +
144 + _check_content_length(response.content_length, max_bytes, media_download_limit)
145 + chunks: list[bytes] = []
146 + received_bytes = 0
147 + async for chunk in response.content.iter_chunked(_RESPONSE_READ_CHUNK_SIZE):
148 + received_bytes = _append_chunk(
149 + chunks, chunk, received_bytes, max_bytes, media_download_limit
150 + )
151 + return b"".join(chunks)
152
153
154 def _is_retryable(exc: Exception) -> bool:
@@ -277,14 +401,22 @@ class HTTPConnection:
401
402 @_sync_retry
403 def get_bytes(
280 - self, url: str, *, timeout: float | None = None, allow_redirects: bool = True
404 + self,
405 + url: str,
406 + *,
407 + timeout: float | None = None,
408 + allow_redirects: bool = True,
409 + max_bytes: int | None = None,
410 ) -> bytes:
411 with self.get_response(
283 - url, timeout=timeout, allow_redirects=allow_redirects
412 + url,
413 + stream=True,
414 + timeout=timeout,
415 + allow_redirects=allow_redirects,
416 ) as r:
417 r.raise_for_status()
418
287 - return r.content
419 + return _read_response_bytes(r, max_bytes)
420
421 @_async_retry
422 async def async_get_bytes(
@@ -293,13 +425,16 @@ class HTTPConnection:
425 *,
426 timeout: float | None = None,
427 allow_redirects: bool = True,
428 + max_bytes: int | None = None,
429 ) -> bytes:
430 async with await self.get_async_response(
298 - url, timeout=timeout, allow_redirects=allow_redirects
431 + url,
432 + timeout=timeout,
433 + allow_redirects=allow_redirects,
434 ) as r:
435 r.raise_for_status()
436
302 - return await r.read()
437 + return await _async_read_response_bytes(r, max_bytes)
438
439 def get_text(self, url: str, *, timeout: float | None = None) -> str:
440 with self.get_response(url, timeout=timeout) as r:
vllm/entrypoints/openai/run_batch.py
+26 -5
@@ -10,7 +10,7 @@ from argparse import Namespace
10 from collections.abc import Awaitable, Callable
11 from http import HTTPStatus
12 from io import BytesIO, StringIO
13 -from typing import Any, TypeAlias
13 +from typing import Any, NoReturn, TypeAlias
14 from urllib.parse import urlparse
15
16 import aiohttp
@@ -28,7 +28,7 @@ from urllib3.util import parse_url
28
29 import vllm.envs as envs
30 from vllm.config import config
31 -from vllm.connections import global_http_connection
31 +from vllm.connections import HTTPResponseSizeExceededError, global_http_connection
32 from vllm.engine.arg_utils import AsyncEngineArgs
33 from vllm.engine.protocol import EngineClient
34 from vllm.entrypoints.launchers.api_server.entry import init_app_state
@@ -68,11 +68,24 @@ from vllm.logger import init_logger
68 from vllm.reasoning import ReasoningParserManager
69 from vllm.utils import random_uuid
70 from vllm.utils.argparse_utils import FlexibleArgumentParser
71 +from vllm.utils.mem_constants import MiB_bytes
72 from vllm.version import __version__ as VLLM_VERSION
73
74 logger = init_logger(__name__)
75
76
77 +def _get_audio_download_limit_bytes() -> int:
78 + return int(envs.VLLM_MAX_AUDIO_CLIP_FILESIZE_MB * MiB_bytes)
79 +
80 +
81 +def _raise_audio_filesize_limit(size_bytes: int) -> NoReturn:
82 + raise VLLMValidationError(
83 + "Maximum file size exceeded",
84 + parameter="audio_filesize_mb",
85 + value=size_bytes / MiB_bytes,
86 + )
87 +
88 +
89 class BatchTranscriptionRequest(TranscriptionRequest):
90 """
91 Batch transcription request that uses file_url instead of file.
@@ -474,6 +487,9 @@ async def download_bytes_from_url(
487 if "," in url:
488 header, data = url.split(",", 1)
489 if "base64" in header:
490 + decoded_size_bytes = len(data.rstrip("=")) * 3 // 4
491 + if decoded_size_bytes > _get_audio_download_limit_bytes():
492 + _raise_audio_filesize_limit(decoded_size_bytes)
493 return base64.b64decode(data)
494 else:
495 raise VLLMValidationError(
@@ -502,9 +518,14 @@ async def download_bytes_from_url(
518 # between urllib3 and aiohttp (e.g. backslash-@ attacks).
519 url = url_spec.url
520
505 - return await global_http_connection.async_get_bytes(
506 - url, allow_redirects=envs.VLLM_MEDIA_URL_ALLOW_REDIRECTS
507 - )
521 + try:
522 + return await global_http_connection.async_get_bytes(
523 + url,
524 + allow_redirects=envs.VLLM_MEDIA_URL_ALLOW_REDIRECTS,
525 + max_bytes=_get_audio_download_limit_bytes(),
526 + )
527 + except HTTPResponseSizeExceededError as exc:
528 + _raise_audio_filesize_limit(exc.received_bytes)
529
530 else:
531 raise VLLMValidationError(
vllm/entrypoints/speech_to_text/base/serving.py
-8
@@ -124,7 +124,6 @@ class SpeechToTextBaseServing(GenerateBaseServing):
124
125 self.enable_force_include_usage = enable_force_include_usage
126
127 - self.max_audio_filesize_mb = envs.VLLM_MAX_AUDIO_CLIP_FILESIZE_MB
127 self.max_audio_decode_duration_s: int = envs.VLLM_MAX_AUDIO_DECODE_DURATION_S
128 self.max_audio_decode_bytes: int = envs.VLLM_MAX_AUDIO_DECODE_BYTES
129 if self.model_cls.supports_segment_timestamp:
@@ -274,13 +273,6 @@ class SpeechToTextBaseServing(GenerateBaseServing):
273 else None
274 )
275
277 - if len(audio_data) / 1024**2 > self.max_audio_filesize_mb:
278 - raise VLLMValidationError(
279 - "Maximum file size exceeded",
280 - parameter="audio_filesize_mb",
281 - value=len(audio_data) / 1024**2,
282 - )
283 -
276 # Run cpu intensive preprocess step in a separate thread pool executor.
277 chunks, duration = await self._decode_and_chunk_speech_async(audio_data)
278
vllm/envs.py
+7
@@ -75,6 +75,7 @@ if TYPE_CHECKING:
75 VLLM_MEDIA_CACHE_MAX_SIZE_MB: int = 5120
76 VLLM_MEDIA_CACHE_TTL_HOURS: float = 24
77 VLLM_MEDIA_FETCH_MAX_RETRIES: int = 3
78 + VLLM_MAX_MEDIA_DOWNLOAD_SIZE_MB: int = 256
79 VLLM_MEDIA_URL_ALLOW_REDIRECTS: bool = True
80 VLLM_MEDIA_LOADING_THREAD_COUNT: int = 8
81 VLLM_MAX_AUDIO_CLIP_FILESIZE_MB: int = 25
@@ -988,6 +989,12 @@ environment_variables: dict[str, Callable[[], Any]] = {
989 "VLLM_MEDIA_FETCH_MAX_RETRIES": lambda: int(
990 os.getenv("VLLM_MEDIA_FETCH_MAX_RETRIES", "3")
991 ),
992 + # Maximum size in MB for a single remote media download. The limit is
993 + # enforced while streaming the response body so oversized or infinite
994 + # responses cannot grow the API server heap without bound. Default is 256.
995 + "VLLM_MAX_MEDIA_DOWNLOAD_SIZE_MB": lambda: int(
996 + os.getenv("VLLM_MAX_MEDIA_DOWNLOAD_SIZE_MB", "256")
997 + ),
998 # Whether to allow HTTP redirects when fetching from media URLs.
999 # Default to True
1000 "VLLM_MEDIA_URL_ALLOW_REDIRECTS": lambda: bool(
vllm/multimodal/media/audio.py
+21
@@ -10,6 +10,7 @@ import pybase64
10 import torch
11
12 import vllm.envs as envs
13 +from vllm.exceptions import VLLMValidationError
14 from vllm.logger import init_logger
15 from vllm.multimodal.audio import resample_audio_pyav
16 from vllm.utils.import_utils import PlaceholderModule
@@ -275,7 +276,20 @@ class AudioMediaIO(MediaIO[tuple[npt.NDArray, float]]):
276 # for flexible control.
277 self.kwargs = kwargs
278
279 + def get_max_bytes(self) -> int:
280 + return int(envs.VLLM_MAX_AUDIO_CLIP_FILESIZE_MB * MiB_bytes)
281 +
282 + def _validate_encoded_size(self, size: int) -> None:
283 + max_bytes = self.get_max_bytes()
284 + if size > max_bytes:
285 + raise VLLMValidationError(
286 + "Maximum file size exceeded",
287 + parameter="audio_filesize_mb",
288 + value=size / MiB_bytes,
289 + )
290 +
291 def load_bytes(self, data: bytes) -> tuple[npt.NDArray, float]:
292 + self._validate_encoded_size(len(data))
293 return load_audio(
294 BytesIO(data),
295 sr=None,
@@ -288,6 +302,13 @@ class AudioMediaIO(MediaIO[tuple[npt.NDArray, float]]):
302 media_type: str,
303 data: str,
304 ) -> tuple[npt.NDArray, float]:
305 + max_encoded_chars = 4 * ((self.get_max_bytes() + 2) // 3)
306 + if len(data) > max_encoded_chars:
307 + raise VLLMValidationError(
308 + "Maximum file size exceeded",
309 + parameter="audio_filesize_mb",
310 + value=(len(data) * 3 / 4) / MiB_bytes,
311 + )
312 return self.load_bytes(pybase64.b64decode(data))
313
314 def load_file(self, filepath: Path) -> tuple[npt.NDArray, float]:
vllm/multimodal/media/base.py
+4
@@ -79,6 +79,10 @@ class MediaIO(ABC, Generic[_T]):
79 merged.update(runtime_kwargs)
80 return merged
81
82 + def get_max_bytes(self) -> int | None:
83 + """Return the maximum encoded payload size accepted by this media IO."""
84 + return None
85 +
86 @abstractmethod
87 def load_bytes(self, data: bytes) -> _T:
88 raise NotImplementedError
vllm/multimodal/media/connector.py
+20 -2
@@ -22,8 +22,12 @@ from PIL import Image, UnidentifiedImageError
22 from urllib3.util import Url, parse_url
23
24 import vllm.envs as envs
25 -from vllm.connections import HTTPConnection, global_http_connection
26 -from vllm.exceptions import VLLMUnprocessableEntityError
25 +from vllm.connections import (
26 + HTTPConnection,
27 + MediaDownloadSizeExceededError,
28 + global_http_connection,
29 +)
30 +from vllm.exceptions import VLLMUnprocessableEntityError, VLLMValidationError
31 from vllm.logger import init_logger
32 from vllm.multimodal.video import get_video_loader_backend_for_processor
33 from vllm.utils.registry import ExtensionManager
@@ -70,6 +74,9 @@ def _wrap_media_fetch_error(
74 Original exception for transient errors (5xx, 408, 429, network blips)
75 or other exceptions
76 """
77 + if isinstance(exc, VLLMValidationError):
78 + return exc
79 +
80 if isinstance(exc, aiohttp.ClientResponseError):
81 if exc.status in (408, 429):
82 return exc
@@ -101,6 +108,13 @@ def _wrap_media_fetch_error(
108 value=url,
109 )
110
111 + if isinstance(exc, MediaDownloadSizeExceededError):
112 + return VLLMUnprocessableEntityError(
113 + f"Failed to fetch media from URL: {exc}",
114 + parameter="image_url",
115 + value=url,
116 + )
117 +
118 if isinstance(exc, ValueError):
119 return VLLMUnprocessableEntityError(
120 "Failed to fetch media from URL: Invalid URL",
@@ -363,6 +377,7 @@ class MediaConnector:
377
378 if url_spec.scheme and url_spec.scheme.startswith("http"):
379 self._assert_url_in_allowed_media_domains(url_spec)
380 + max_bytes = media_io.get_max_bytes()
381
382 cached = self._get_cached_bytes(url)
383 if cached is not None:
@@ -374,6 +389,7 @@ class MediaConnector:
389 url_spec.url,
390 timeout=fetch_timeout,
391 allow_redirects=envs.VLLM_MEDIA_URL_ALLOW_REDIRECTS,
392 + max_bytes=max_bytes,
393 )
394 except Exception as e:
395 wrapped = _wrap_media_fetch_error(url, e)
@@ -409,6 +425,7 @@ class MediaConnector:
425
426 if url_spec.scheme and url_spec.scheme.startswith("http"):
427 self._assert_url_in_allowed_media_domains(url_spec)
428 + max_bytes = media_io.get_max_bytes()
429
430 cached = await loop.run_in_executor(
431 global_thread_pool, self._get_cached_bytes, url
@@ -425,6 +442,7 @@ class MediaConnector:
442 url_spec.url,
443 timeout=fetch_timeout,
444 allow_redirects=envs.VLLM_MEDIA_URL_ALLOW_REDIRECTS,
445 + max_bytes=max_bytes,
446 )
447 except Exception as e:
448 wrapped = _wrap_media_fetch_error(url, e)