A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 19:45 UTC
[Bugfix] Clean up multi-modal processors (#14417)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Mar 7, 2025 at 18:33 UTC
05fb6718f0d80519fcb8011ccd841fc7f37db3c1
12 files changed
+79
-76
vllm/config.py
+9
@@ -2405,6 +2405,15 @@ class MultiModalConfig:
2405
hash_str = hashlib.md5(str(factors).encode()).hexdigest()
2406
return hash_str
2407
2408
+ def get_limit_per_prompt(self, modality: str) -> int:
2409
+ """
2410
+ Get the maximum number of input items allowed per prompt
2411
+ for the given modality.
2412
+
2413
+ If not set by the user, this defaults to `1`.
2414
+ """
2415
+ return self.limit_per_prompt.get(modality, 1)
2416
+
2417
# TODO: Add configs to init vision tower or not.
2418
2419
vllm/model_executor/models/deepseek_vl2.py
+27
-29
@@ -14,7 +14,6 @@ from einops import rearrange, repeat
14
from transformers import BatchFeature
15
16
from vllm.config import VllmConfig
17
-from vllm.logger import init_logger
17
from vllm.model_executor import SamplingMetadata
18
from vllm.model_executor.layers.quantization import QuantizationConfig
19
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
@@ -25,8 +24,8 @@ from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
24
from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
25
ImageSize, MultiModalDataItems)
26
from vllm.multimodal.processing import (BaseMultiModalProcessor,
28
- BaseProcessingInfo, ProcessingCache,
29
- PromptReplacement, PromptUpdate)
27
+ BaseProcessingInfo, PromptReplacement,
28
+ PromptUpdate)
29
from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
30
from vllm.sequence import IntermediateTensors
31
from vllm.transformers_utils.configs.deepseek_vl2 import (DeepseekVLV2Config,
@@ -42,8 +41,6 @@ from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,
41
init_vllm_registered_model, maybe_prefix,
42
merge_multimodal_embeddings)
43
45
-logger = init_logger(__name__)
46
-
44
# The image token id may be various
45
_IMAGE_TOKEN = "<image>"
46
@@ -216,30 +213,6 @@ class DeepseekVL2DummyInputsBuilder(
213
class DeepseekVL2MultiModalProcessor(
214
BaseMultiModalProcessor[DeepseekVL2ProcessingInfo]):
215
219
- def __init__(
220
- self,
221
- info: DeepseekVL2ProcessingInfo,
222
- dummy_inputs: "BaseDummyInputsBuilder[DeepseekVL2ProcessingInfo]",
223
- *,
224
- cache: Optional[ProcessingCache] = None,
225
- enable_sanity_checks: bool = True) -> None:
226
- super().__init__(
227
- info,
228
- dummy_inputs,
229
- cache=cache,
230
- enable_sanity_checks=enable_sanity_checks,
231
- )
232
-
233
- mm_limit = self.info.ctx.model_config.multimodal_config.limit_per_prompt
234
- if self.cache is not None and mm_limit["image"] > 2:
235
- # The processor output depends on the number of images passed,
236
- # making it incompatible with processing cache which is supposed
237
- # to be invariant of how many images are passed per prompt
238
- self.cache = None
239
- logger.warning_once(
240
- f"{type(self).__name__} does not support processing cache with "
241
- "image limit larger than 2.")
242
-
216
def _call_hf_processor(
217
self,
218
prompt: str,
@@ -316,6 +289,31 @@ class DeepseekVL2MultiModalProcessor(
289
)
290
]
291
292
+ def _cached_apply_hf_processor(
293
+ self,
294
+ prompt: Union[str, list[int]],
295
+ mm_data_items: MultiModalDataItems,
296
+ hf_processor_mm_kwargs: Mapping[str, object],
297
+ ) -> tuple[list[int], MultiModalKwargs, bool]:
298
+ # The processor logic is different for len(images) <= 2 vs > 2
299
+ # Since the processing cache assumes that the processor output is
300
+ # invariant of how many images are passed per prompt, we only
301
+ # perform caching for the most common case
302
+ if mm_data_items.get_count("image", strict=False) > 2:
303
+ # This code path corresponds to the cache being disabled
304
+ return self._apply_hf_processor_main(
305
+ prompt=prompt,
306
+ mm_items=mm_data_items,
307
+ hf_processor_mm_kwargs=hf_processor_mm_kwargs,
308
+ enable_hf_prompt_update=True,
309
+ )
310
+
311
+ return super()._cached_apply_hf_processor(
312
+ prompt=prompt,
313
+ mm_data_items=mm_data_items,
314
+ hf_processor_mm_kwargs=hf_processor_mm_kwargs,
315
+ )
316
+
317
318
@MULTIMODAL_REGISTRY.register_processor(
319
DeepseekVL2MultiModalProcessor,
vllm/model_executor/models/h2ovl.py
+28
-30
@@ -8,21 +8,19 @@
8
# Licensed under Apache 2.0 License [see LICENSE for details]
9
# --------------------------------------------------------
10
from collections.abc import Mapping, Sequence
11
-from typing import Optional
11
+from typing import Optional, Union
12
13
import torch
14
from PIL import Image
15
from transformers import PretrainedConfig
16
17
-from vllm.logger import init_logger
17
from vllm.model_executor.layers.quantization import QuantizationConfig
18
from vllm.multimodal import MULTIMODAL_REGISTRY
19
from vllm.multimodal.inputs import MultiModalKwargs
20
from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
21
MultiModalDataItems)
23
-from vllm.multimodal.processing import (ProcessingCache, PromptReplacement,
24
- PromptUpdate, PromptUpdateDetails)
25
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
22
+from vllm.multimodal.processing import (PromptReplacement, PromptUpdate,
23
+ PromptUpdateDetails)
24
from vllm.transformers_utils.tokenizer import AnyTokenizer
25
26
from .intern_vit import InternVisionModel
@@ -32,8 +30,6 @@ from .internvl import (IMG_CONTEXT, IMG_END, IMG_START,
30
InternVLMultiModalProcessor, build_transform,
31
find_closest_aspect_ratio, get_internvl_target_ratios)
32
35
-logger = init_logger(__name__)
36
-
33
34
def resolve_h2ovl_min_max_num(
35
*,
@@ -465,29 +461,6 @@ class H2OVLProcessingInfo(BaseInternVLProcessingInfo):
461
class H2OVLMultiModalProcessor(InternVLMultiModalProcessor[H2OVLProcessingInfo]
462
):
463
468
- def __init__(self,
469
- info: H2OVLProcessingInfo,
470
- dummy_inputs: "BaseDummyInputsBuilder[H2OVLProcessingInfo]",
471
- *,
472
- cache: Optional[ProcessingCache] = None,
473
- enable_sanity_checks: bool = True) -> None:
474
- super().__init__(
475
- info,
476
- dummy_inputs,
477
- cache=cache,
478
- enable_sanity_checks=enable_sanity_checks,
479
- )
480
-
481
- mm_limit = self.info.ctx.model_config.multimodal_config.limit_per_prompt
482
- if self.cache is not None and mm_limit["image"] >= 2:
483
- # The processor output depends on the number of images passed,
484
- # making it incompatible with processing cache which is supposed
485
- # to be invariant of how many images are passed per prompt
486
- self.cache = None
487
- logger.warning_once(
488
- f"{type(self).__name__} does not support processing cache with "
489
- "multi-image support enabled.")
490
-
464
def _get_prompt_updates(
465
self,
466
mm_items: MultiModalDataItems,
@@ -543,6 +516,31 @@ class H2OVLMultiModalProcessor(InternVLMultiModalProcessor[H2OVLProcessingInfo]
516
)
517
]
518
519
+ def _cached_apply_hf_processor(
520
+ self,
521
+ prompt: Union[str, list[int]],
522
+ mm_data_items: MultiModalDataItems,
523
+ hf_processor_mm_kwargs: Mapping[str, object],
524
+ ) -> tuple[list[int], MultiModalKwargs, bool]:
525
+ # The processor logic is different for len(images) <= 1 vs > 1
526
+ # Since the processing cache assumes that the processor output is
527
+ # invariant of how many images are passed per prompt, we only
528
+ # perform caching for the most common case
529
+ if mm_data_items.get_count("image", strict=False) > 1:
530
+ # This code path corresponds to the cache being disabled
531
+ return self._apply_hf_processor_main(
532
+ prompt=prompt,
533
+ mm_items=mm_data_items,
534
+ hf_processor_mm_kwargs=hf_processor_mm_kwargs,
535
+ enable_hf_prompt_update=True,
536
+ )
537
+
538
+ return super()._cached_apply_hf_processor(
539
+ prompt=prompt,
540
+ mm_data_items=mm_data_items,
541
+ hf_processor_mm_kwargs=hf_processor_mm_kwargs,
542
+ )
543
+
544
545
@MULTIMODAL_REGISTRY.register_processor(
546
H2OVLMultiModalProcessor,
vllm/model_executor/models/llava_next_video.py
+1
-1
@@ -133,7 +133,7 @@ class LlavaNextVideoProcessingInfo(BaseProcessingInfo):
133
134
def get_num_frames_with_most_features(self, seq_len: int) -> int:
135
mm_config = self.ctx.get_mm_config()
136
- max_videos = mm_config.limit_per_prompt.get("video", 1)
136
+ max_videos = mm_config.get_limit_per_prompt("video")
137
138
max_total_frames = self._get_max_video_frames(seq_len)
139
vllm/model_executor/models/llava_onevision.py
+2
-2
@@ -206,8 +206,8 @@ class LlavaOnevisionProcessingInfo(LlavaNextProcessingInfo):
206
207
def get_num_frames_with_most_features(self, seq_len: int) -> int:
208
mm_config = self.ctx.get_mm_config()
209
- max_images = mm_config.limit_per_prompt.get("image", 1)
210
- max_videos = mm_config.limit_per_prompt.get("video", 1)
209
+ max_images = mm_config.get_limit_per_prompt("image")
210
+ max_videos = mm_config.get_limit_per_prompt("video")
211
212
max_image_tokens = self.get_max_image_tokens() * max_images
213
max_total_frames = self._get_max_video_frames(seq_len -
vllm/model_executor/models/minicpmo.py
+3
-3
@@ -201,9 +201,9 @@ class MiniCPMOProcessingInfo(MiniCPMVProcessingInfo):
201
202
def get_num_frames_with_most_features(self, seq_len: int) -> int:
203
mm_config = self.ctx.get_mm_config()
204
- max_images = mm_config.limit_per_prompt.get("image", 1)
205
- max_videos = mm_config.limit_per_prompt.get("video", 1)
206
- max_audios = mm_config.limit_per_prompt.get("audio", 1)
204
+ max_images = mm_config.get_limit_per_prompt("image")
205
+ max_videos = mm_config.get_limit_per_prompt("video")
206
+ max_audios = mm_config.get_limit_per_prompt("audio")
207
208
# count <image_idx></image_idx> tokens
209
# which are not in get_max_image_tokens
vllm/model_executor/models/minicpmv.py
+2
-2
@@ -446,8 +446,8 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
446
447
def get_num_frames_with_most_features(self, seq_len: int) -> int:
448
mm_config = self.ctx.get_mm_config()
449
- max_images = mm_config.limit_per_prompt.get("image", 1)
450
- max_videos = mm_config.limit_per_prompt.get("video", 1)
449
+ max_images = mm_config.get_limit_per_prompt("image")
450
+ max_videos = mm_config.get_limit_per_prompt("video")
451
452
# count <image_idx></image_idx> tokens
453
# which are not in get_max_image_tokens
vllm/model_executor/models/pixtral.py
+1
-1
@@ -68,7 +68,7 @@ def dummy_data_for_pixtral(ctx: InputContext, seq_len: int,
68
image_token_id = mm_encoder.special_ids.img
69
70
mm_config = ctx.get_mm_config()
71
- num_images = mm_config.limit_per_prompt.get("image", 1)
71
+ num_images = mm_config.get_limit_per_prompt("image")
72
73
# dummy size
74
size = 256
vllm/model_executor/models/qwen2_vl.py
+2
-2
@@ -911,8 +911,8 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
911
912
def get_num_frames_with_most_features(self, seq_len: int) -> int:
913
mm_config = self.ctx.get_mm_config()
914
- max_images = mm_config.limit_per_prompt.get("image", 1)
915
- max_videos = mm_config.limit_per_prompt.get("video", 1)
914
+ max_images = mm_config.get_limit_per_prompt("image")
915
+ max_videos = mm_config.get_limit_per_prompt("video")
916
917
max_image_tokens = self.get_max_image_tokens() * max_images
918
max_total_frames = self._get_max_video_frames(seq_len -
vllm/multimodal/processing.py
+2
-2
@@ -984,10 +984,10 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
984
before passing them to :meth:`_get_hf_mm_data`.
985
"""
986
mm_items = self.data_parser.parse_mm_data(mm_data)
987
+ mm_config = self.info.ctx.get_mm_config()
988
988
- mm_limits = self.info.ctx.get_mm_config().limit_per_prompt
989
for modality, items in mm_items.items():
990
- limit = mm_limits.get(modality, 1)
990
+ limit = mm_config.get_limit_per_prompt(modality)
991
if len(items) > limit:
992
raise ValueError(
993
f"You set {modality}={limit} (or defaulted to 1) in "
vllm/multimodal/profiling.py
+1
-3
@@ -110,12 +110,10 @@ class MultiModalProfiler(Generic[_I]):
110
111
def get_mm_limits(self) -> Mapping[str, int]:
112
mm_config = self.processing_info.ctx.get_mm_config()
113
- mm_limit_per_prompt = mm_config.limit_per_prompt
114
-
113
supported_mm_limits = self.processing_info.get_supported_mm_limits()
114
115
mm_limits = {
118
- modality: mm_limit_per_prompt.get(modality, 1)
116
+ modality: mm_config.get_limit_per_prompt(modality)
117
for modality in supported_mm_limits
118
}
119
vllm/multimodal/registry.py
+1
-1
@@ -355,7 +355,7 @@ class MultiModalRegistry:
355
# TODO: Automatically determine the limits based on budget
356
# once more models support multi-image inputs
357
limits_per_plugin = {
358
- key: config_limits_per_plugin.get(key, 1)
358
+ key: multimodal_config.get_limit_per_prompt(key)
359
for key in self._plugins
360
}
361