A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 2, 06:00 UTC
[Multimodal] Automatically select registered video loader for VLM (#44126)
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Isotr0py committed
Jun 2, 2026 at 17:09 UTC
f8e9c56d1587b2eb5599800aa907e091ec9a8182
5 files changed
+188
-9
tests/multimodal/test_video.py
+48
@@ -10,8 +10,12 @@ import pytest
10
from vllm.assets.base import get_vllm_public_assets
11
from vllm.multimodal.video import (
12
VIDEO_LOADER_REGISTRY,
13
+ DynamicVideoBackend,
14
+ Molmo2VideoBackend,
15
VideoLoader,
16
+ get_video_loader_backend_for_processor,
17
)
18
+from vllm.transformers_utils.processor import get_video_processor_cls_name_from_config
19
20
from .utils import create_long_gop_video, create_video_from_image
21
@@ -54,6 +58,50 @@ def test_video_loader_type_doesnt_exist():
58
VIDEO_LOADER_REGISTRY.load("non_existing_video_loader")
59
60
61
+# ============================================================================
62
+# Video Processor → Video Loader Tests (via model repo)
63
+# ============================================================================
64
+
65
+
66
+@pytest.mark.parametrize(
67
+ "model_repo, expected_loader_cls",
68
+ [
69
+ pytest.param(
70
+ "allenai/Molmo2-4B",
71
+ Molmo2VideoBackend,
72
+ id="molmo2",
73
+ ),
74
+ pytest.param(
75
+ "zai-org/GLM-4.1V-9B-Thinking",
76
+ DynamicVideoBackend,
77
+ id="glm4v",
78
+ ),
79
+ ],
80
+)
81
+def test_video_processor_from_model_repo(
82
+ model_repo: str,
83
+ expected_loader_cls: type,
84
+):
85
+ """Test that a model repo resolves to the correct video loader backend.
86
+
87
+ The test downloads the preprocessor config from HuggingFace Hub,
88
+ extracts the ``video_processor_type`` field, and verifies it maps
89
+ to the expected backend and loader class.
90
+ """
91
+ video_processor = get_video_processor_cls_name_from_config(model_repo)
92
+ assert video_processor is not None, (
93
+ f"Model repo {model_repo!r} did not contain a video_processor_type "
94
+ f"in its preprocessor config"
95
+ )
96
+
97
+ backend = get_video_loader_backend_for_processor(video_processor)
98
+ loader = VIDEO_LOADER_REGISTRY.load(backend)
99
+ assert isinstance(loader, expected_loader_cls), (
100
+ f"{model_repo!r}: backend={backend!r} loaded "
101
+ f"{type(loader)}, expected {expected_loader_cls}"
102
+ )
103
+
104
+
105
def test_video_backend_handles_broken_frames(monkeypatch: pytest.MonkeyPatch):
106
"""
107
Regression test for handling videos with broken frames.
vllm/entrypoints/chat_utils.py
+19
-2
@@ -58,6 +58,7 @@ from vllm.renderers.embed_utils import (
58
safe_load_prompt_embeds,
59
safe_load_prompt_embeds_async,
60
)
61
+from vllm.transformers_utils.processor import get_video_processor_cls_name
62
from vllm.utils import random_uuid
63
from vllm.utils.collection_utils import is_list_of
64
from vllm.utils.import_utils import LazyLoader
@@ -577,6 +578,10 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
578
def mm_processor(self):
579
return self.mm_registry.create_processor(self.model_config)
580
581
+ @property
582
+ def video_processor_name(self) -> str | None:
583
+ return get_video_processor_cls_name(self.model_config)
584
+
585
def add(self, modality: ModalityStr, item: _T) -> str | None:
586
"""
587
Add a multi-modal item to the current prompt and returns the
@@ -1025,7 +1030,14 @@ class MultiModalContentParser(BaseMultiModalContentParser):
1030
return self.parse_audio(audio_url, uuid)
1031
1032
def parse_video(self, video_url: str | None, uuid: str | None = None) -> None:
1028
- video = self._connector.fetch_video(video_url=video_url) if video_url else None
1033
+ video = (
1034
+ self._connector.fetch_video(
1035
+ video_url=video_url,
1036
+ video_processor=self._tracker.video_processor_name,
1037
+ )
1038
+ if video_url
1039
+ else None
1040
+ )
1041
1042
placeholder = self._tracker.add("video", (video, uuid))
1043
self._add_placeholder("video", placeholder)
@@ -1205,7 +1217,12 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
1217
1218
async def _video_with_uuid_async(self, video_url: str | None, uuid: str | None):
1219
video = (
1208
- await self._connector.fetch_video_async(video_url) if video_url else None
1220
+ await self._connector.fetch_video_async(
1221
+ video_url,
1222
+ video_processor=self._tracker.video_processor_name,
1223
+ )
1224
+ if video_url
1225
+ else None
1226
)
1227
return video, uuid
1228
vllm/multimodal/media/connector.py
+15
-2
@@ -22,6 +22,7 @@ from urllib3.util import Url, parse_url
22
import vllm.envs as envs
23
from vllm.connections import HTTPConnection, global_http_connection
24
from vllm.logger import init_logger
25
+from vllm.multimodal.video import get_video_loader_backend_for_processor
26
from vllm.utils.registry import ExtensionManager
27
28
from .audio import AudioEmbeddingMediaIO, AudioMediaIO
@@ -452,6 +453,7 @@ class MediaConnector:
453
video_url: str,
454
*,
455
image_mode: str = "RGB",
456
+ video_processor: str | None = None,
457
) -> tuple[npt.NDArray, dict[str, Any]]:
458
"""
459
Load video from an HTTP or base64 data URL.
@@ -459,7 +461,12 @@ class MediaConnector:
461
image_io = ImageMediaIO(
462
image_mode=image_mode, **self.media_io_kwargs.get("image", {})
463
)
462
- video_io = VideoMediaIO(image_io, **self.media_io_kwargs.get("video", {}))
464
+ video_io_kwargs = dict(self.media_io_kwargs.get("video", {}))
465
+ if "video_backend" not in video_io_kwargs and (
466
+ video_backend := get_video_loader_backend_for_processor(video_processor)
467
+ ):
468
+ video_io_kwargs["video_backend"] = video_backend
469
+ video_io = VideoMediaIO(image_io, **video_io_kwargs)
470
471
return self.load_from_url(
472
video_url,
@@ -472,6 +479,7 @@ class MediaConnector:
479
video_url: str,
480
*,
481
image_mode: str = "RGB",
482
+ video_processor: str | None = None,
483
) -> tuple[npt.NDArray, dict[str, Any]]:
484
"""
485
Asynchronously load video from an HTTP or base64 data URL.
@@ -481,7 +489,12 @@ class MediaConnector:
489
image_io = ImageMediaIO(
490
image_mode=image_mode, **self.media_io_kwargs.get("image", {})
491
)
484
- video_io = VideoMediaIO(image_io, **self.media_io_kwargs.get("video", {}))
492
+ video_io_kwargs = dict(self.media_io_kwargs.get("video", {}))
493
+ if "video_backend" not in video_io_kwargs and (
494
+ video_backend := get_video_loader_backend_for_processor(video_processor)
495
+ ):
496
+ video_io_kwargs["video_backend"] = video_backend
497
+ video_io = VideoMediaIO(image_io, **video_io_kwargs)
498
499
return await self.load_from_url_async(
500
video_url,
vllm/multimodal/video.py
+68
-5
@@ -28,6 +28,60 @@ except ImportError:
28
logger = init_logger(__name__)
29
30
31
+class VideoLoaderRegistry(ExtensionManager):
32
+ def __init__(self) -> None:
33
+ super().__init__()
34
+ self.processor2backend: dict[str, str] = {}
35
+
36
+ @staticmethod
37
+ def _normalize_registered_video_processors(
38
+ video_processor: str | tuple[str, ...] | None,
39
+ ) -> tuple[str, ...]:
40
+ if video_processor is None:
41
+ return ()
42
+
43
+ if isinstance(video_processor, str):
44
+ return (video_processor,)
45
+
46
+ if all(isinstance(processor, str) for processor in video_processor):
47
+ return video_processor
48
+
49
+ raise TypeError(
50
+ "video_processor must be a class name or a tuple of class names"
51
+ )
52
+
53
+ def register(
54
+ self,
55
+ name: str,
56
+ *,
57
+ video_processor: str | tuple[str, ...] | None = None,
58
+ ):
59
+ processors = self._normalize_registered_video_processors(video_processor)
60
+
61
+ def wrap(cls_to_register):
62
+ self.name2class[name] = cls_to_register
63
+ for processor_name in processors:
64
+ self.processor2backend[processor_name] = name
65
+ return cls_to_register
66
+
67
+ return wrap
68
+
69
+ def get_backend_for_video_processor(
70
+ self,
71
+ video_processor: str | None,
72
+ ) -> str | None:
73
+ if video_processor is None:
74
+ return None
75
+
76
+ return self.processor2backend.get(video_processor)
77
+
78
+
79
+def get_video_loader_backend_for_processor(
80
+ video_processor: str | None,
81
+) -> str | None:
82
+ return VIDEO_LOADER_REGISTRY.get_backend_for_video_processor(video_processor)
83
+
84
+
85
def resize_video(frames: npt.NDArray, size: tuple[int, int]) -> npt.NDArray:
86
num_frames, _, _, channels = frames.shape
87
new_height, new_width = size
@@ -113,7 +167,7 @@ class VideoLoader:
167
}
168
169
116
-VIDEO_LOADER_REGISTRY = ExtensionManager()
170
+VIDEO_LOADER_REGISTRY = VideoLoaderRegistry()
171
172
173
class OpenCVVideoBackendMixin:
@@ -550,7 +604,10 @@ class VideoBackend(VideoLoader, OpenCVVideoBackendMixin, PyAVVideoBackendMixin):
604
)
605
606
553
-@VIDEO_LOADER_REGISTRY.register("opencv_dynamic")
607
+@VIDEO_LOADER_REGISTRY.register(
608
+ "opencv_dynamic",
609
+ video_processor="Glm4vVideoProcessor",
610
+)
611
class DynamicVideoBackend(VideoBackend):
612
"""Duration-aware dynamic-sampling video backend.
613
@@ -639,8 +696,11 @@ class DynamicVideoBackend(VideoBackend):
696
)
697
698
642
-@VIDEO_LOADER_REGISTRY.register("glm4_6v")
643
-class GLM4_6VVideoBackend(VideoBackend):
699
+@VIDEO_LOADER_REGISTRY.register(
700
+ "glmga",
701
+ video_processor="GlmgaVideoProcessor",
702
+)
703
+class GLMGAVideoBackend(VideoBackend):
704
@classmethod
705
def _prepare_source(cls, source: VideoSourceMetadata) -> VideoSourceMetadata:
706
# Estimate duration from frame count and fps when the container
@@ -740,7 +800,10 @@ class GLM4_6VVideoBackend(VideoBackend):
800
return frames, metadata
801
802
743
-@VIDEO_LOADER_REGISTRY.register("molmo2")
803
+@VIDEO_LOADER_REGISTRY.register(
804
+ "molmo2",
805
+ video_processor="Molmo2VideoProcessor",
806
+)
807
class Molmo2VideoBackend(VideoLoader, OpenCVVideoBackendMixin):
808
@classmethod
809
def get_candidate_target_fps(
vllm/transformers_utils/processor.py
+38
@@ -161,6 +161,44 @@ def get_processor_cls_name_from_config(
161
return None
162
163
164
+def get_video_processor_cls_name_from_config(
165
+ processor_name: str,
166
+ revision: str | None = "main",
167
+) -> str | None:
168
+ processor_name = convert_model_repo_to_path(processor_name)
169
+ config_file = [
170
+ "video_preprocessor_config.json",
171
+ "preprocessor_config.json",
172
+ ]
173
+ for file in config_file:
174
+ config = get_hf_file_to_dict(file, processor_name, revision=revision)
175
+ if config and "video_processor_type" in config:
176
+ return config["video_processor_type"]
177
+ return None
178
+
179
+
180
+_cached_get_video_processor_cls_name = lru_cache(
181
+ get_video_processor_cls_name_from_config
182
+)
183
+
184
+
185
+def get_video_processor_cls_name(
186
+ model_config: "ModelConfig",
187
+) -> str | None:
188
+ if is_gguf(model_config.model):
189
+ assert not is_gguf(model_config.tokenizer), (
190
+ "For multimodal GGUF models, the original tokenizer "
191
+ "should be used to correctly load video processor metadata."
192
+ )
193
+ model = model_config.tokenizer
194
+ revision = model_config.tokenizer_revision
195
+ else:
196
+ model = model_config.model
197
+ revision = model_config.revision
198
+
199
+ return _cached_get_video_processor_cls_name(model, revision=revision)
200
+
201
+
202
def get_processor(
203
processor_name: str,
204
*args: Any,