A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Frontend] Multithreaded async multimodal load_bytes (#22710)
Signed-off-by: Alexandre Milesi <30204471+milesial@users.noreply.github.com> Co-authored-by: Alexandre Milesi <30204471+milesial@users.noreply.github.com>
milesial committed
Aug 13, 2025 at 06:09 UTC
20d65aa75548c4ee0c9b69847f177dec085cd358
2 files changed
+27
-6
vllm/envs.py
+7
@@ -63,6 +63,7 @@ if TYPE_CHECKING:
63
VLLM_IMAGE_FETCH_TIMEOUT: int = 5
64
VLLM_VIDEO_FETCH_TIMEOUT: int = 30
65
VLLM_AUDIO_FETCH_TIMEOUT: int = 10
66
+ VLLM_MEDIA_LOADING_THREAD_COUNT: int = 8
67
VLLM_MAX_AUDIO_CLIP_FILESIZE_MB: int = 25
68
VLLM_VIDEO_LOADER_BACKEND: str = "opencv"
69
VLLM_MM_INPUT_CACHE_GIB: int = 4
@@ -555,6 +556,12 @@ environment_variables: dict[str, Callable[[], Any]] = {
556
"VLLM_AUDIO_FETCH_TIMEOUT":
557
lambda: int(os.getenv("VLLM_AUDIO_FETCH_TIMEOUT", "10")),
558
559
+ # Max number of workers for the thread pool handling
560
+ # media bytes loading. Set to 1 to disable parallel processing.
561
+ # Default is 8
562
+ "VLLM_MEDIA_LOADING_THREAD_COUNT":
563
+ lambda: int(os.getenv("VLLM_MEDIA_LOADING_THREAD_COUNT", "8")),
564
+
565
# Maximum filesize in MB for a single audio file when processing
566
# speech-to-text requests. Files larger than this will be rejected.
567
# Default is 25 MB
vllm/multimodal/utils.py
+20
-6
@@ -1,6 +1,9 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
+import asyncio
5
+import atexit
6
+from concurrent.futures import ThreadPoolExecutor
7
from itertools import groupby
8
from pathlib import Path
9
from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union
@@ -33,6 +36,10 @@ else:
36
MultiModalKwargs = Any
37
MultiModalPlaceholderDict = Any
38
39
+global_thread_pool = ThreadPoolExecutor(
40
+ max_workers=envs.VLLM_MEDIA_LOADING_THREAD_COUNT)
41
+atexit.register(global_thread_pool.shutdown)
42
+
43
44
class MediaConnector:
45
@@ -139,19 +146,26 @@ class MediaConnector:
146
fetch_timeout: Optional[int] = None,
147
) -> _M:
148
url_spec = urlparse(url)
149
+ loop = asyncio.get_running_loop()
150
151
if url_spec.scheme.startswith("http"):
152
connection = self.connection
153
data = await connection.async_get_bytes(url, timeout=fetch_timeout)
146
-
147
- return media_io.load_bytes(data)
154
+ future = loop.run_in_executor(global_thread_pool,
155
+ media_io.load_bytes, data)
156
+ return await future
157
158
if url_spec.scheme == "data":
150
- return self._load_data_url(url_spec, media_io)
159
+ future = loop.run_in_executor(global_thread_pool,
160
+ self._load_data_url, url_spec,
161
+ media_io)
162
+ return await future
163
164
if url_spec.scheme == "file":
153
- return self._load_file_url(url_spec, media_io)
154
-
165
+ future = loop.run_in_executor(global_thread_pool,
166
+ self._load_file_url, url_spec,
167
+ media_io)
168
+ return await future
169
msg = "The URL must be either a HTTP, data or file URL."
170
raise ValueError(msg)
171
@@ -489,4 +503,4 @@ def fetch_video(
503
"video": video_io_kwargs
504
}
505
media_connector = MediaConnector(media_io_kwargs=media_io_kwargs)
492
- return media_connector.fetch_video(video_url)
\ No newline at end of file
506
+ return media_connector.fetch_video(video_url)