A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 13, 19:30 UTC
[Multimodal] Make MediaConnector extensible. (#27759)
Signed-off-by: Chenheli Hua <huachenheli@outlook.com>
Chenheli Hua committed
Nov 4, 2025 at 10:28 UTC
1fb4217a052189feb8709b67bb3209ab316d13b7
5 files changed
+71
-22
vllm/entrypoints/chat_utils.py
+7
-3
@@ -43,11 +43,12 @@ from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast, Processor
43
# pydantic needs the TypedDict from typing_extensions
44
from typing_extensions import Required, TypedDict
45
46
+from vllm import envs
47
from vllm.config import ModelConfig
48
from vllm.logger import init_logger
49
from vllm.model_executor.models import SupportsMultiModal
50
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalDataDict, MultiModalUUIDDict
50
-from vllm.multimodal.utils import MediaConnector
51
+from vllm.multimodal.utils import MEDIA_CONNECTOR_REGISTRY, MediaConnector
52
from vllm.transformers_utils.chat_templates import get_chat_template_fallback_path
53
from vllm.transformers_utils.processor import cached_get_processor
54
from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
@@ -806,7 +807,9 @@ class MultiModalContentParser(BaseMultiModalContentParser):
807
self._tracker = tracker
808
multimodal_config = self._tracker.model_config.multimodal_config
809
media_io_kwargs = getattr(multimodal_config, "media_io_kwargs", None)
809
- self._connector = MediaConnector(
810
+
811
+ self._connector: MediaConnector = MEDIA_CONNECTOR_REGISTRY.load(
812
+ envs.VLLM_MEDIA_CONNECTOR,
813
media_io_kwargs=media_io_kwargs,
814
allowed_local_media_path=tracker.allowed_local_media_path,
815
allowed_media_domains=tracker.allowed_media_domains,
@@ -891,7 +894,8 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
894
self._tracker = tracker
895
multimodal_config = self._tracker.model_config.multimodal_config
896
media_io_kwargs = getattr(multimodal_config, "media_io_kwargs", None)
894
- self._connector = MediaConnector(
897
+ self._connector: MediaConnector = MEDIA_CONNECTOR_REGISTRY.load(
898
+ envs.VLLM_MEDIA_CONNECTOR,
899
media_io_kwargs=media_io_kwargs,
900
allowed_local_media_path=tracker.allowed_local_media_path,
901
allowed_media_domains=tracker.allowed_media_domains,
vllm/envs.py
+9
@@ -70,6 +70,7 @@ if TYPE_CHECKING:
70
VLLM_MEDIA_LOADING_THREAD_COUNT: int = 8
71
VLLM_MAX_AUDIO_CLIP_FILESIZE_MB: int = 25
72
VLLM_VIDEO_LOADER_BACKEND: str = "opencv"
73
+ VLLM_MEDIA_CONNECTOR: str = "http"
74
VLLM_MM_INPUT_CACHE_GIB: int = 4
75
VLLM_TARGET_DEVICE: str = "cuda"
76
VLLM_MAIN_CUDA_VERSION: str = "12.8"
@@ -738,6 +739,14 @@ environment_variables: dict[str, Callable[[], Any]] = {
739
"VLLM_VIDEO_LOADER_BACKEND": lambda: os.getenv(
740
"VLLM_VIDEO_LOADER_BACKEND", "opencv"
741
),
742
+ # Media connector implementation.
743
+ # - "http": Default connector that supports fetching media via HTTP.
744
+ #
745
+ # Custom implementations can be registered
746
+ # via `@MEDIA_CONNECTOR_REGISTRY.register("my_custom_media_connector")` and
747
+ # imported at runtime.
748
+ # If a non-existing backend is used, an AssertionError will be thrown.
749
+ "VLLM_MEDIA_CONNECTOR": lambda: os.getenv("VLLM_MEDIA_CONNECTOR", "http"),
750
# [DEPRECATED] Cache size (in GiB per process) for multimodal input cache
751
# Default is 4 GiB per API process + 4 GiB per engine core process
752
"VLLM_MM_INPUT_CACHE_GIB": lambda: int(os.getenv("VLLM_MM_INPUT_CACHE_GIB", "4")),
vllm/multimodal/utils.py
+4
@@ -20,6 +20,7 @@ import vllm.envs as envs
20
from vllm.connections import HTTPConnection, global_http_connection
21
from vllm.logger import init_logger
22
from vllm.utils.jsontree import json_map_leaves
23
+from vllm.utils.registry import ExtensionManager
24
25
from .audio import AudioMediaIO
26
from .base import MediaIO
@@ -46,7 +47,10 @@ atexit.register(global_thread_pool.shutdown)
47
48
_M = TypeVar("_M")
49
50
+MEDIA_CONNECTOR_REGISTRY = ExtensionManager()
51
52
+
53
+@MEDIA_CONNECTOR_REGISTRY.register("http")
54
class MediaConnector:
55
def __init__(
56
self,
vllm/multimodal/video.py
+2
-19
@@ -14,6 +14,7 @@ from PIL import Image
14
15
from vllm import envs
16
from vllm.logger import init_logger
17
+from vllm.utils.registry import ExtensionManager
18
19
from .base import MediaIO
20
from .image import ImageMediaIO
@@ -63,25 +64,7 @@ class VideoLoader:
64
raise NotImplementedError
65
66
66
-class VideoLoaderRegistry:
67
- def __init__(self) -> None:
68
- self.name2class: dict[str, type] = {}
69
-
70
- def register(self, name: str):
71
- def wrap(cls_to_register):
72
- self.name2class[name] = cls_to_register
73
- return cls_to_register
74
-
75
- return wrap
76
-
77
- @staticmethod
78
- def load(cls_name: str) -> VideoLoader:
79
- cls = VIDEO_LOADER_REGISTRY.name2class.get(cls_name)
80
- assert cls is not None, f"VideoLoader class {cls_name} not found"
81
- return cls()
82
-
83
-
84
-VIDEO_LOADER_REGISTRY = VideoLoaderRegistry()
67
+VIDEO_LOADER_REGISTRY = ExtensionManager()
68
69
70
@VIDEO_LOADER_REGISTRY.register("opencv")
vllm/utils/registry.py
new
+49
@@ -0,0 +1,49 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+from typing import Any
4
+
5
+
6
+class ExtensionManager:
7
+ """
8
+ A registry for managing pluggable extension classes.
9
+
10
+ This class provides a simple mechanism to register and instantiate
11
+ extension classes by name. It is commonly used to implement plugin
12
+ systems where different implementations can be swapped at runtime.
13
+
14
+ Examples:
15
+ Basic usage with a registry instance:
16
+
17
+ >>> FOO_REGISTRY = ExtensionManager()
18
+ >>> @FOO_REGISTRY.register("my_foo_impl")
19
+ ... class MyFooImpl(Foo):
20
+ ... def __init__(self, value):
21
+ ... self.value = value
22
+ >>> foo_impl = FOO_REGISTRY.load("my_foo_impl", value=123)
23
+
24
+ """
25
+
26
+ def __init__(self) -> None:
27
+ """
28
+ Initialize an empty extension registry.
29
+ """
30
+ self.name2class: dict[str, type] = {}
31
+
32
+ def register(self, name: str):
33
+ """
34
+ Decorator to register a class with the given name.
35
+ """
36
+
37
+ def wrap(cls_to_register):
38
+ self.name2class[name] = cls_to_register
39
+ return cls_to_register
40
+
41
+ return wrap
42
+
43
+ def load(self, cls_name: str, *args, **kwargs) -> Any:
44
+ """
45
+ Instantiate and return a registered extension class by name.
46
+ """
47
+ cls = self.name2class.get(cls_name)
48
+ assert cls is not None, f"Extension class {cls_name} not found"
49
+ return cls(*args, **kwargs)