A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 9, 10:00 UTC
[Chore] Move `MediaConnector` to `vllm.multimodal.media` (#33324)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Jan 30, 2026 at 00:54 UTC
831453fcef69f76f3266ed2a0d8f193b86e786ad
8 files changed
+381
-350
tests/multimodal/test_utils.py
+2
-1
@@ -14,7 +14,8 @@ from PIL import Image, ImageChops
14
15
from vllm.multimodal.image import convert_image_mode
16
from vllm.multimodal.inputs import PlaceholderRange
17
-from vllm.multimodal.utils import MediaConnector, argsort_mm_positions
17
+from vllm.multimodal.media import MediaConnector
18
+from vllm.multimodal.utils import argsort_mm_positions
19
20
# Test different image extensions (JPG/PNG) and formats (gray/RGB/RGBA)
21
TEST_IMAGE_ASSETS = [
vllm/entrypoints/chat_utils.py
+1
-1
@@ -50,8 +50,8 @@ from vllm.multimodal.inputs import (
50
VisionChunkImage,
51
VisionChunkVideo,
52
)
53
+from vllm.multimodal.media import MEDIA_CONNECTOR_REGISTRY, MediaConnector
54
from vllm.multimodal.processing import BaseMultiModalProcessor
54
-from vllm.multimodal.utils import MEDIA_CONNECTOR_REGISTRY, MediaConnector
55
from vllm.utils import random_uuid
56
from vllm.utils.collection_utils import is_list_of
57
from vllm.utils.import_utils import LazyLoader
vllm/model_executor/models/nemotron_parse.py
+1
-1
@@ -58,8 +58,8 @@ from vllm.multimodal.processing import (
58
PromptReplacement,
59
PromptUpdate,
60
)
61
+from vllm.tokenizers import TokenizerLike
62
from vllm.transformers_utils.configs.radio import RadioConfig
62
-from vllm.transformers_utils.tokenizer import TokenizerLike
63
from vllm.utils.tensor_schema import TensorSchema, TensorShape
64
from vllm.v1.attention.backend import AttentionType
65
vllm/multimodal/inputs.py
+3
-3
@@ -27,12 +27,12 @@ from vllm.utils.collection_utils import is_list_of
27
from vllm.utils.import_utils import LazyLoader
28
from vllm.utils.jsontree import json_map_leaves
29
30
+from .media import MediaWithBytes
31
+
32
if TYPE_CHECKING:
33
import torch
34
import torch.types
35
from transformers.feature_extraction_utils import BatchFeature
34
-
35
- from .media import MediaWithBytes
36
else:
37
torch = LazyLoader("torch", globals(), "torch")
38
@@ -58,7 +58,7 @@ Represents a single audio
58
item, which can be passed to a HuggingFace `AudioProcessor`.
59
"""
60
61
-ImageItem: TypeAlias = Union[HfImageItem, "torch.Tensor", "MediaWithBytes[HfImageItem]"]
61
+ImageItem: TypeAlias = Union[HfImageItem, "torch.Tensor", MediaWithBytes[HfImageItem]]
62
"""
63
A `transformers.image_utils.ImageInput` representing a single image
64
item, which can be passed to a HuggingFace `ImageProcessor`.
vllm/multimodal/media/__init__.py
+5
-1
@@ -2,8 +2,9 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
from .audio import AudioEmbeddingMediaIO, AudioMediaIO
4
from .base import MediaIO, MediaWithBytes
5
+from .connector import MEDIA_CONNECTOR_REGISTRY, MediaConnector
6
from .image import ImageEmbeddingMediaIO, ImageMediaIO
6
-from .video import VideoMediaIO
7
+from .video import VIDEO_LOADER_REGISTRY, VideoMediaIO
8
9
__all__ = [
10
"MediaIO",
@@ -12,5 +13,8 @@ __all__ = [
13
"AudioMediaIO",
14
"ImageEmbeddingMediaIO",
15
"ImageMediaIO",
16
+ "VIDEO_LOADER_REGISTRY",
17
"VideoMediaIO",
18
+ "MEDIA_CONNECTOR_REGISTRY",
19
+ "MediaConnector",
20
]
vllm/multimodal/media/connector.py
new
+343
@@ -0,0 +1,343 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+import asyncio
5
+import atexit
6
+from concurrent.futures import ThreadPoolExecutor
7
+from pathlib import Path
8
+from typing import Any, TypeVar
9
+from urllib.request import url2pathname
10
+
11
+import numpy as np
12
+import numpy.typing as npt
13
+import torch
14
+from PIL import Image, UnidentifiedImageError
15
+from urllib3.util import Url, parse_url
16
+
17
+import vllm.envs as envs
18
+from vllm.connections import HTTPConnection, global_http_connection
19
+from vllm.utils.registry import ExtensionManager
20
+
21
+from .audio import AudioEmbeddingMediaIO, AudioMediaIO
22
+from .base import MediaIO
23
+from .image import ImageEmbeddingMediaIO, ImageMediaIO
24
+from .video import VideoMediaIO
25
+
26
+_M = TypeVar("_M")
27
+
28
+global_thread_pool = ThreadPoolExecutor(
29
+ max_workers=envs.VLLM_MEDIA_LOADING_THREAD_COUNT
30
+)
31
+atexit.register(global_thread_pool.shutdown)
32
+
33
+MEDIA_CONNECTOR_REGISTRY = ExtensionManager()
34
+
35
+
36
+@MEDIA_CONNECTOR_REGISTRY.register("http")
37
+class MediaConnector:
38
+ def __init__(
39
+ self,
40
+ media_io_kwargs: dict[str, dict[str, Any]] | None = None,
41
+ connection: HTTPConnection = global_http_connection,
42
+ *,
43
+ allowed_local_media_path: str = "",
44
+ allowed_media_domains: list[str] | None = None,
45
+ ) -> None:
46
+ """
47
+ Args:
48
+ media_io_kwargs: Additional args passed to process media
49
+ inputs, keyed by modalities. For example,
50
+ to set num_frames for video, set
51
+ `--media-io-kwargs '{"video":{"num_frames":40}}'`
52
+ connection: HTTP connection client to download media contents.
53
+ allowed_local_media_path: A local directory to load media files from.
54
+ allowed_media_domains: If set, only media URLs that belong to this
55
+ domain can be used for multi-modal inputs.
56
+ """
57
+ super().__init__()
58
+
59
+ self.media_io_kwargs: dict[str, dict[str, Any]] = (
60
+ media_io_kwargs if media_io_kwargs else {}
61
+ )
62
+ self.connection = connection
63
+
64
+ if allowed_local_media_path:
65
+ allowed_local_media_path_ = Path(allowed_local_media_path)
66
+
67
+ if not allowed_local_media_path_.exists():
68
+ raise ValueError(
69
+ "Invalid `--allowed-local-media-path`: The path "
70
+ f"{allowed_local_media_path_} does not exist."
71
+ )
72
+ if not allowed_local_media_path_.is_dir():
73
+ raise ValueError(
74
+ "Invalid `--allowed-local-media-path`: The path "
75
+ f"{allowed_local_media_path_} must be a directory."
76
+ )
77
+ else:
78
+ allowed_local_media_path_ = None
79
+
80
+ self.allowed_local_media_path = allowed_local_media_path_
81
+ if allowed_media_domains is None:
82
+ allowed_media_domains = []
83
+ self.allowed_media_domains = allowed_media_domains
84
+
85
+ def _load_data_url(
86
+ self,
87
+ url_spec: Url,
88
+ media_io: MediaIO[_M],
89
+ ) -> _M: # type: ignore[type-var]
90
+ url_spec_path = url_spec.path or ""
91
+ data_spec, data = url_spec_path.split(",", 1)
92
+ media_type, data_type = data_spec.split(";", 1)
93
+ # media_type starts with a leading "/" (e.g., "/video/jpeg")
94
+ media_type = media_type.lstrip("/")
95
+
96
+ if data_type != "base64":
97
+ msg = "Only base64 data URLs are supported for now."
98
+ raise NotImplementedError(msg)
99
+
100
+ return media_io.load_base64(media_type, data)
101
+
102
+ def _load_file_url(
103
+ self,
104
+ url_spec: Url,
105
+ media_io: MediaIO[_M],
106
+ ) -> _M: # type: ignore[type-var]
107
+ allowed_local_media_path = self.allowed_local_media_path
108
+ if allowed_local_media_path is None:
109
+ raise RuntimeError(
110
+ "Cannot load local files without `--allowed-local-media-path`."
111
+ )
112
+
113
+ url_spec_path = url_spec.path or ""
114
+ url_spec_netloc = url_spec.netloc or ""
115
+ filepath = Path(url2pathname(url_spec_netloc + url_spec_path))
116
+ if allowed_local_media_path not in filepath.resolve().parents:
117
+ raise ValueError(
118
+ f"The file path {filepath} must be a subpath "
119
+ f"of `--allowed-local-media-path {allowed_local_media_path}`."
120
+ )
121
+
122
+ return media_io.load_file(filepath)
123
+
124
+ def _assert_url_in_allowed_media_domains(self, url_spec: Url) -> None:
125
+ if (
126
+ self.allowed_media_domains
127
+ and url_spec.hostname not in self.allowed_media_domains
128
+ ):
129
+ raise ValueError(
130
+ f"The URL must be from one of the allowed domains: "
131
+ f"{self.allowed_media_domains}. Input URL domain: "
132
+ f"{url_spec.hostname}"
133
+ )
134
+
135
+ def load_from_url(
136
+ self,
137
+ url: str,
138
+ media_io: MediaIO[_M],
139
+ *,
140
+ fetch_timeout: int | None = None,
141
+ ) -> _M: # type: ignore[type-var]
142
+ url_spec = parse_url(url)
143
+
144
+ if url_spec.scheme and url_spec.scheme.startswith("http"):
145
+ self._assert_url_in_allowed_media_domains(url_spec)
146
+
147
+ connection = self.connection
148
+ data = connection.get_bytes(
149
+ url,
150
+ timeout=fetch_timeout,
151
+ allow_redirects=envs.VLLM_MEDIA_URL_ALLOW_REDIRECTS,
152
+ )
153
+
154
+ return media_io.load_bytes(data)
155
+
156
+ if url_spec.scheme == "data":
157
+ return self._load_data_url(url_spec, media_io)
158
+
159
+ if url_spec.scheme == "file":
160
+ return self._load_file_url(url_spec, media_io)
161
+
162
+ msg = "The URL must be either a HTTP, data or file URL."
163
+ raise ValueError(msg)
164
+
165
+ async def load_from_url_async(
166
+ self,
167
+ url: str,
168
+ media_io: MediaIO[_M],
169
+ *,
170
+ fetch_timeout: int | None = None,
171
+ ) -> _M:
172
+ url_spec = parse_url(url)
173
+ loop = asyncio.get_running_loop()
174
+
175
+ if url_spec.scheme and url_spec.scheme.startswith("http"):
176
+ self._assert_url_in_allowed_media_domains(url_spec)
177
+
178
+ connection = self.connection
179
+ data = await connection.async_get_bytes(
180
+ url,
181
+ timeout=fetch_timeout,
182
+ allow_redirects=envs.VLLM_MEDIA_URL_ALLOW_REDIRECTS,
183
+ )
184
+ future = loop.run_in_executor(global_thread_pool, media_io.load_bytes, data)
185
+ return await future
186
+
187
+ if url_spec.scheme == "data":
188
+ future = loop.run_in_executor(
189
+ global_thread_pool, self._load_data_url, url_spec, media_io
190
+ )
191
+ return await future
192
+
193
+ if url_spec.scheme == "file":
194
+ future = loop.run_in_executor(
195
+ global_thread_pool, self._load_file_url, url_spec, media_io
196
+ )
197
+ return await future
198
+ msg = "The URL must be either a HTTP, data or file URL."
199
+ raise ValueError(msg)
200
+
201
+ def fetch_audio(
202
+ self,
203
+ audio_url: str,
204
+ ) -> tuple[np.ndarray, int | float]:
205
+ """
206
+ Load audio from a URL.
207
+ """
208
+ audio_io = AudioMediaIO(**self.media_io_kwargs.get("audio", {}))
209
+
210
+ return self.load_from_url(
211
+ audio_url,
212
+ audio_io,
213
+ fetch_timeout=envs.VLLM_AUDIO_FETCH_TIMEOUT,
214
+ )
215
+
216
+ async def fetch_audio_async(
217
+ self,
218
+ audio_url: str,
219
+ ) -> tuple[np.ndarray, int | float]:
220
+ """
221
+ Asynchronously fetch audio from a URL.
222
+ """
223
+ audio_io = AudioMediaIO(**self.media_io_kwargs.get("audio", {}))
224
+
225
+ return await self.load_from_url_async(
226
+ audio_url,
227
+ audio_io,
228
+ fetch_timeout=envs.VLLM_AUDIO_FETCH_TIMEOUT,
229
+ )
230
+
231
+ def fetch_image(
232
+ self,
233
+ image_url: str,
234
+ *,
235
+ image_mode: str = "RGB",
236
+ ) -> Image.Image:
237
+ """
238
+ Load a PIL image from an HTTP or base64 data URL.
239
+
240
+ By default, the image is converted into RGB format.
241
+ """
242
+ image_io = ImageMediaIO(
243
+ image_mode=image_mode, **self.media_io_kwargs.get("image", {})
244
+ )
245
+
246
+ try:
247
+ return self.load_from_url(
248
+ image_url,
249
+ image_io,
250
+ fetch_timeout=envs.VLLM_IMAGE_FETCH_TIMEOUT,
251
+ )
252
+ except UnidentifiedImageError as e:
253
+ # convert to ValueError to be properly caught upstream
254
+ raise ValueError(str(e)) from e
255
+
256
+ async def fetch_image_async(
257
+ self,
258
+ image_url: str,
259
+ *,
260
+ image_mode: str = "RGB",
261
+ ) -> Image.Image:
262
+ """
263
+ Asynchronously load a PIL image from an HTTP or base64 data URL.
264
+
265
+ By default, the image is converted into RGB format.
266
+ """
267
+ image_io = ImageMediaIO(
268
+ image_mode=image_mode, **self.media_io_kwargs.get("image", {})
269
+ )
270
+
271
+ try:
272
+ return await self.load_from_url_async(
273
+ image_url,
274
+ image_io,
275
+ fetch_timeout=envs.VLLM_IMAGE_FETCH_TIMEOUT,
276
+ )
277
+ except UnidentifiedImageError as e:
278
+ # convert to ValueError to be properly caught upstream
279
+ raise ValueError(str(e)) from e
280
+
281
+ def fetch_video(
282
+ self,
283
+ video_url: str,
284
+ *,
285
+ image_mode: str = "RGB",
286
+ ) -> tuple[npt.NDArray, dict[str, Any]]:
287
+ """
288
+ Load video from an HTTP or base64 data URL.
289
+ """
290
+ image_io = ImageMediaIO(
291
+ image_mode=image_mode, **self.media_io_kwargs.get("image", {})
292
+ )
293
+ video_io = VideoMediaIO(image_io, **self.media_io_kwargs.get("video", {}))
294
+
295
+ return self.load_from_url(
296
+ video_url,
297
+ video_io,
298
+ fetch_timeout=envs.VLLM_VIDEO_FETCH_TIMEOUT,
299
+ )
300
+
301
+ async def fetch_video_async(
302
+ self,
303
+ video_url: str,
304
+ *,
305
+ image_mode: str = "RGB",
306
+ ) -> tuple[npt.NDArray, dict[str, Any]]:
307
+ """
308
+ Asynchronously load video from an HTTP or base64 data URL.
309
+
310
+ By default, the image is converted into RGB format.
311
+ """
312
+ image_io = ImageMediaIO(
313
+ image_mode=image_mode, **self.media_io_kwargs.get("image", {})
314
+ )
315
+ video_io = VideoMediaIO(image_io, **self.media_io_kwargs.get("video", {}))
316
+
317
+ return await self.load_from_url_async(
318
+ video_url,
319
+ video_io,
320
+ fetch_timeout=envs.VLLM_VIDEO_FETCH_TIMEOUT,
321
+ )
322
+
323
+ def fetch_image_embedding(
324
+ self,
325
+ data: str,
326
+ ) -> torch.Tensor:
327
+ """
328
+ Load image embedding from a URL.
329
+ """
330
+ image_embedding_io = ImageEmbeddingMediaIO()
331
+
332
+ return image_embedding_io.load_base64("", data)
333
+
334
+ def fetch_audio_embedding(
335
+ self,
336
+ data: str,
337
+ ) -> torch.Tensor:
338
+ """
339
+ Load audio embedding from a URL.
340
+ """
341
+ audio_embedding_io = AudioEmbeddingMediaIO()
342
+
343
+ return audio_embedding_io.load_base64("", data)
vllm/multimodal/utils.py
+24
-343
@@ -1,367 +1,50 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
-import asyncio
5
-import atexit
4
import mimetypes
5
+import warnings
6
from collections.abc import Generator
8
-from concurrent.futures import ThreadPoolExecutor
7
from itertools import groupby
10
-from pathlib import Path
11
-from typing import TYPE_CHECKING, Any, TypeVar
12
-from urllib.request import url2pathname
8
+from typing import TYPE_CHECKING, Any
9
10
import numpy as np
11
import numpy.typing as npt
16
-import torch
17
-from PIL import Image, UnidentifiedImageError
18
-from urllib3.util import Url, parse_url
12
+from PIL import Image
13
20
-import vllm.envs as envs
21
-from vllm.connections import HTTPConnection, global_http_connection
14
from vllm.logger import init_logger
23
-from vllm.utils.registry import ExtensionManager
24
-
25
-from .media import (
26
- AudioEmbeddingMediaIO,
27
- AudioMediaIO,
28
- ImageEmbeddingMediaIO,
29
- ImageMediaIO,
30
- MediaIO,
31
- VideoMediaIO,
15
+from vllm.utils.import_utils import LazyLoader
16
+
17
+from .inputs import (
18
+ BatchedTensorInputs,
19
+ MultiModalKwargsItem,
20
+ MultiModalKwargsItems,
21
+ MultiModalPlaceholderDict,
22
)
23
+from .media import AudioMediaIO, ImageMediaIO, MediaConnector, VideoMediaIO
24
25
if TYPE_CHECKING:
35
- from .inputs import (
36
- BatchedTensorInputs,
37
- MultiModalKwargsItem,
38
- MultiModalPlaceholderDict,
39
- )
26
+ import torch.types
27
else:
41
- BatchedTensorInputs = Any
42
- MultiModalKwargsItem = Any
43
- MultiModalPlaceholderDict = Any
28
+ torch = LazyLoader("torch", globals(), "torch")
29
30
logger = init_logger(__name__)
31
47
-global_thread_pool = ThreadPoolExecutor(
48
- max_workers=envs.VLLM_MEDIA_LOADING_THREAD_COUNT
49
-)
50
-atexit.register(global_thread_pool.shutdown)
51
-
52
-_M = TypeVar("_M")
53
-
54
-MEDIA_CONNECTOR_REGISTRY = ExtensionManager()
55
-
56
-
57
-@MEDIA_CONNECTOR_REGISTRY.register("http")
58
-class MediaConnector:
59
- def __init__(
60
- self,
61
- media_io_kwargs: dict[str, dict[str, Any]] | None = None,
62
- connection: HTTPConnection = global_http_connection,
63
- *,
64
- allowed_local_media_path: str = "",
65
- allowed_media_domains: list[str] | None = None,
66
- ) -> None:
67
- """
68
- Args:
69
- media_io_kwargs: Additional args passed to process media
70
- inputs, keyed by modalities. For example,
71
- to set num_frames for video, set
72
- `--media-io-kwargs '{"video":{"num_frames":40}}'`
73
- connection: HTTP connection client to download media contents.
74
- allowed_local_media_path: A local directory to load media files from.
75
- allowed_media_domains: If set, only media URLs that belong to this
76
- domain can be used for multi-modal inputs.
77
- """
78
- super().__init__()
79
-
80
- self.media_io_kwargs: dict[str, dict[str, Any]] = (
81
- media_io_kwargs if media_io_kwargs else {}
82
- )
83
- self.connection = connection
84
-
85
- if allowed_local_media_path:
86
- allowed_local_media_path_ = Path(allowed_local_media_path)
87
-
88
- if not allowed_local_media_path_.exists():
89
- raise ValueError(
90
- "Invalid `--allowed-local-media-path`: The path "
91
- f"{allowed_local_media_path_} does not exist."
92
- )
93
- if not allowed_local_media_path_.is_dir():
94
- raise ValueError(
95
- "Invalid `--allowed-local-media-path`: The path "
96
- f"{allowed_local_media_path_} must be a directory."
97
- )
98
- else:
99
- allowed_local_media_path_ = None
100
-
101
- self.allowed_local_media_path = allowed_local_media_path_
102
- if allowed_media_domains is None:
103
- allowed_media_domains = []
104
- self.allowed_media_domains = allowed_media_domains
105
-
106
- def _load_data_url(
107
- self,
108
- url_spec: Url,
109
- media_io: MediaIO[_M],
110
- ) -> _M: # type: ignore[type-var]
111
- url_spec_path = url_spec.path or ""
112
- data_spec, data = url_spec_path.split(",", 1)
113
- media_type, data_type = data_spec.split(";", 1)
114
- # media_type starts with a leading "/" (e.g., "/video/jpeg")
115
- media_type = media_type.lstrip("/")
116
-
117
- if data_type != "base64":
118
- msg = "Only base64 data URLs are supported for now."
119
- raise NotImplementedError(msg)
120
-
121
- return media_io.load_base64(media_type, data)
122
-
123
- def _load_file_url(
124
- self,
125
- url_spec: Url,
126
- media_io: MediaIO[_M],
127
- ) -> _M: # type: ignore[type-var]
128
- allowed_local_media_path = self.allowed_local_media_path
129
- if allowed_local_media_path is None:
130
- raise RuntimeError(
131
- "Cannot load local files without `--allowed-local-media-path`."
132
- )
133
-
134
- url_spec_path = url_spec.path or ""
135
- url_spec_netloc = url_spec.netloc or ""
136
- filepath = Path(url2pathname(url_spec_netloc + url_spec_path))
137
- if allowed_local_media_path not in filepath.resolve().parents:
138
- raise ValueError(
139
- f"The file path {filepath} must be a subpath "
140
- f"of `--allowed-local-media-path {allowed_local_media_path}`."
141
- )
142
-
143
- return media_io.load_file(filepath)
144
-
145
- def _assert_url_in_allowed_media_domains(self, url_spec: Url) -> None:
146
- if (
147
- self.allowed_media_domains
148
- and url_spec.hostname not in self.allowed_media_domains
149
- ):
150
- raise ValueError(
151
- f"The URL must be from one of the allowed domains: "
152
- f"{self.allowed_media_domains}. Input URL domain: "
153
- f"{url_spec.hostname}"
154
- )
155
-
156
- def load_from_url(
157
- self,
158
- url: str,
159
- media_io: MediaIO[_M],
160
- *,
161
- fetch_timeout: int | None = None,
162
- ) -> _M: # type: ignore[type-var]
163
- url_spec = parse_url(url)
164
-
165
- if url_spec.scheme and url_spec.scheme.startswith("http"):
166
- self._assert_url_in_allowed_media_domains(url_spec)
167
-
168
- connection = self.connection
169
- data = connection.get_bytes(
170
- url,
171
- timeout=fetch_timeout,
172
- allow_redirects=envs.VLLM_MEDIA_URL_ALLOW_REDIRECTS,
173
- )
174
-
175
- return media_io.load_bytes(data)
176
-
177
- if url_spec.scheme == "data":
178
- return self._load_data_url(url_spec, media_io)
179
-
180
- if url_spec.scheme == "file":
181
- return self._load_file_url(url_spec, media_io)
182
-
183
- msg = "The URL must be either a HTTP, data or file URL."
184
- raise ValueError(msg)
185
-
186
- async def load_from_url_async(
187
- self,
188
- url: str,
189
- media_io: MediaIO[_M],
190
- *,
191
- fetch_timeout: int | None = None,
192
- ) -> _M:
193
- url_spec = parse_url(url)
194
- loop = asyncio.get_running_loop()
195
-
196
- if url_spec.scheme and url_spec.scheme.startswith("http"):
197
- self._assert_url_in_allowed_media_domains(url_spec)
198
-
199
- connection = self.connection
200
- data = await connection.async_get_bytes(
201
- url,
202
- timeout=fetch_timeout,
203
- allow_redirects=envs.VLLM_MEDIA_URL_ALLOW_REDIRECTS,
204
- )
205
- future = loop.run_in_executor(global_thread_pool, media_io.load_bytes, data)
206
- return await future
207
-
208
- if url_spec.scheme == "data":
209
- future = loop.run_in_executor(
210
- global_thread_pool, self._load_data_url, url_spec, media_io
211
- )
212
- return await future
213
-
214
- if url_spec.scheme == "file":
215
- future = loop.run_in_executor(
216
- global_thread_pool, self._load_file_url, url_spec, media_io
217
- )
218
- return await future
219
- msg = "The URL must be either a HTTP, data or file URL."
220
- raise ValueError(msg)
221
-
222
- def fetch_audio(
223
- self,
224
- audio_url: str,
225
- ) -> tuple[np.ndarray, int | float]:
226
- """
227
- Load audio from a URL.
228
- """
229
- audio_io = AudioMediaIO(**self.media_io_kwargs.get("audio", {}))
230
-
231
- return self.load_from_url(
232
- audio_url,
233
- audio_io,
234
- fetch_timeout=envs.VLLM_AUDIO_FETCH_TIMEOUT,
235
- )
32
237
- async def fetch_audio_async(
238
- self,
239
- audio_url: str,
240
- ) -> tuple[np.ndarray, int | float]:
241
- """
242
- Asynchronously fetch audio from a URL.
243
- """
244
- audio_io = AudioMediaIO(**self.media_io_kwargs.get("audio", {}))
245
-
246
- return await self.load_from_url_async(
247
- audio_url,
248
- audio_io,
249
- fetch_timeout=envs.VLLM_AUDIO_FETCH_TIMEOUT,
250
- )
33
+def __getattr__(name: str):
34
+ if name == "MEDIA_CONNECTOR_REGISTRY":
35
+ from .media import MEDIA_CONNECTOR_REGISTRY
36
252
- def fetch_image(
253
- self,
254
- image_url: str,
255
- *,
256
- image_mode: str = "RGB",
257
- ) -> Image.Image:
258
- """
259
- Load a PIL image from an HTTP or base64 data URL.
260
-
261
- By default, the image is converted into RGB format.
262
- """
263
- image_io = ImageMediaIO(
264
- image_mode=image_mode, **self.media_io_kwargs.get("image", {})
37
+ warnings.warn(
38
+ "`vllm.multimodal.utils.MEDIA_CONNECTOR_REGISTRY` "
39
+ "has been moved to `vllm.multimodal.media.MEDIA_CONNECTOR_REGISTRY`. "
40
+ "The old name will be removed in v0.17.",
41
+ DeprecationWarning,
42
+ stacklevel=2,
43
)
44
267
- try:
268
- return self.load_from_url(
269
- image_url,
270
- image_io,
271
- fetch_timeout=envs.VLLM_IMAGE_FETCH_TIMEOUT,
272
- )
273
- except UnidentifiedImageError as e:
274
- # convert to ValueError to be properly caught upstream
275
- raise ValueError(str(e)) from e
276
-
277
- async def fetch_image_async(
278
- self,
279
- image_url: str,
280
- *,
281
- image_mode: str = "RGB",
282
- ) -> Image.Image:
283
- """
284
- Asynchronously load a PIL image from an HTTP or base64 data URL.
285
-
286
- By default, the image is converted into RGB format.
287
- """
288
- image_io = ImageMediaIO(
289
- image_mode=image_mode, **self.media_io_kwargs.get("image", {})
290
- )
45
+ return MEDIA_CONNECTOR_REGISTRY
46
292
- try:
293
- return await self.load_from_url_async(
294
- image_url,
295
- image_io,
296
- fetch_timeout=envs.VLLM_IMAGE_FETCH_TIMEOUT,
297
- )
298
- except UnidentifiedImageError as e:
299
- # convert to ValueError to be properly caught upstream
300
- raise ValueError(str(e)) from e
301
-
302
- def fetch_video(
303
- self,
304
- video_url: str,
305
- *,
306
- image_mode: str = "RGB",
307
- ) -> tuple[npt.NDArray, dict[str, Any]]:
308
- """
309
- Load video from an HTTP or base64 data URL.
310
- """
311
- image_io = ImageMediaIO(
312
- image_mode=image_mode, **self.media_io_kwargs.get("image", {})
313
- )
314
- video_io = VideoMediaIO(image_io, **self.media_io_kwargs.get("video", {}))
315
-
316
- return self.load_from_url(
317
- video_url,
318
- video_io,
319
- fetch_timeout=envs.VLLM_VIDEO_FETCH_TIMEOUT,
320
- )
321
-
322
- async def fetch_video_async(
323
- self,
324
- video_url: str,
325
- *,
326
- image_mode: str = "RGB",
327
- ) -> tuple[npt.NDArray, dict[str, Any]]:
328
- """
329
- Asynchronously load video from an HTTP or base64 data URL.
330
-
331
- By default, the image is converted into RGB format.
332
- """
333
- image_io = ImageMediaIO(
334
- image_mode=image_mode, **self.media_io_kwargs.get("image", {})
335
- )
336
- video_io = VideoMediaIO(image_io, **self.media_io_kwargs.get("video", {}))
337
-
338
- return await self.load_from_url_async(
339
- video_url,
340
- video_io,
341
- fetch_timeout=envs.VLLM_VIDEO_FETCH_TIMEOUT,
342
- )
343
-
344
- def fetch_image_embedding(
345
- self,
346
- data: str,
347
- ) -> torch.Tensor:
348
- """
349
- Load image embedding from a URL.
350
- """
351
- image_embedding_io = ImageEmbeddingMediaIO()
352
-
353
- return image_embedding_io.load_base64("", data)
354
-
355
- def fetch_audio_embedding(
356
- self,
357
- data: str,
358
- ) -> torch.Tensor:
359
- """
360
- Load audio embedding from a URL.
361
- """
362
- audio_embedding_io = AudioEmbeddingMediaIO()
363
-
364
- return audio_embedding_io.load_base64("", data)
47
+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
48
49
50
def encode_audio_base64(
@@ -483,8 +166,6 @@ def group_mm_kwargs_by_modality(
166
Yields:
167
A tuple `(modality, num_items, grouped_kwargs)`.
168
"""
486
- from vllm.multimodal.inputs import MultiModalKwargsItems
487
-
169
for modality, group in groupby(mm_kwargs, key=lambda x: x[0]):
170
items_lst = [item for _, item in group]
171
mm_kwargs_items = MultiModalKwargsItems({modality: items_lst})
vllm/transformers_utils/tokenizer.py
+2
@@ -17,3 +17,5 @@ def __getattr__(name: str):
17
)
18
19
return get_tokenizer
20
+
21
+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")