A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 22:35 UTC
[Frontend] Improve error message for too many mm items (#22114)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Aug 2, 2025 at 17:20 UTC
f5d0f4784fdd93f1032f3bb81220af10d7588f5a
5 files changed
+52
-51
tests/entrypoints/test_chat_utils.py
+2
-8
@@ -579,10 +579,7 @@ def test_parse_chat_messages_rejects_too_many_images_in_one_message(
579
warnings.filterwarnings(
580
"ignore",
581
message="coroutine 'async_get_and_parse_image' was never awaited")
582
- with pytest.raises(
583
- ValueError,
584
- match="At most 2 image\\(s\\) may be provided in one request\\."
585
- ):
582
+ with pytest.raises(ValueError, match="At most"):
583
parse_chat_messages(
584
[{
585
"role":
@@ -622,10 +619,7 @@ def test_parse_chat_messages_rejects_too_many_images_across_messages(
619
warnings.filterwarnings(
620
"ignore",
621
message="coroutine 'async_get_and_parse_image' was never awaited")
625
- with pytest.raises(
626
- ValueError,
627
- match="At most 2 image\\(s\\) may be provided in one request\\."
628
- ):
622
+ with pytest.raises(ValueError, match="At most"):
623
parse_chat_messages(
624
[{
625
"role":
tests/multimodal/test_processing.py
+4
-6
@@ -3,7 +3,6 @@
3
4
from contextlib import nullcontext
5
from typing import Optional, cast
6
-from unittest.mock import MagicMock
6
7
import numpy as np
8
import pytest
@@ -957,15 +956,14 @@ def test_limit_mm_per_prompt_dummy(model_id, limit, num_supported, is_valid):
956
)
957
958
processor = MULTIMODAL_REGISTRY.create_processor(model_config)
960
- profiler = MultiModalProfiler(processor)
959
+ processor._supported_mm_limits = {"image": num_supported}
960
962
- mock_supported_mm_limits = MagicMock(return_value={"image": num_supported})
963
- processor.info.get_supported_mm_limits = mock_supported_mm_limits
961
+ profiler = MultiModalProfiler(processor)
962
963
if is_valid:
964
exc_ctx = nullcontext()
965
else:
968
- exc_ctx = pytest.raises(ValueError, match="The model only supports")
966
+ exc_ctx = pytest.raises(ValueError, match="At most")
967
968
with exc_ctx:
969
profiler.get_decoder_dummy_data(
@@ -1002,7 +1000,7 @@ def test_limit_mm_per_prompt_apply(model_id, num_images, limit, is_valid):
1000
if is_valid:
1001
exc_ctx = nullcontext()
1002
else:
1005
- exc_ctx = pytest.raises(ValueError, match=f"passed {num_images} image")
1003
+ exc_ctx = pytest.raises(ValueError, match="At most")
1004
1005
with exc_ctx:
1006
processor.apply(
vllm/entrypoints/chat_utils.py
+10
-17
@@ -535,9 +535,10 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
535
return self._model_config
536
537
@cached_property
538
- def model_cls(self):
538
+ def model_cls(self) -> type[SupportsMultiModal]:
539
from vllm.model_executor.model_loader import get_model_cls
540
- return get_model_cls(self.model_config)
540
+ model_cls = get_model_cls(self.model_config)
541
+ return cast(type[SupportsMultiModal], model_cls)
542
543
@property
544
def allowed_local_media_path(self):
@@ -547,31 +548,23 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
548
def mm_registry(self):
549
return MULTIMODAL_REGISTRY
550
551
+ @cached_property
552
+ def mm_processor(self):
553
+ return self.mm_registry.create_processor(self.model_config)
554
+
555
def add(self, modality: ModalityStr, item: _T) -> Optional[str]:
556
"""
557
Add a multi-modal item to the current prompt and returns the
558
placeholder string to use, if any.
559
"""
555
- mm_registry = self.mm_registry
556
- model_config = self.model_config
557
- model_cls = cast(SupportsMultiModal, self.model_cls)
558
-
560
input_modality = modality.replace("_embeds", "")
561
+ num_items = len(self._items_by_modality[modality]) + 1
562
561
- mm_processor = mm_registry.create_processor(model_config)
562
- allowed_counts = mm_processor.info.get_allowed_mm_limits()
563
- allowed_count = allowed_counts.get(input_modality, 0)
564
-
565
- current_count = len(self._items_by_modality[modality]) + 1
566
- if current_count > allowed_count:
567
- raise ValueError(
568
- f"At most {allowed_count} {modality}(s) may be provided in "
569
- "one request. You can set `--limit-mm-per-prompt` to "
570
- "increase this limit if the model supports it.")
563
+ self.mm_processor.validate_num_items(input_modality, num_items)
564
565
self._items_by_modality[modality].append(item)
566
574
- return model_cls.get_placeholder_str(modality, current_count)
567
+ return self.model_cls.get_placeholder_str(modality, num_items)
568
569
@abstractmethod
570
def create_parser(self) -> "BaseMultiModalContentParser":
vllm/multimodal/processing.py
+35
-19
@@ -1,6 +1,5 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-import json
3
import sys
4
from abc import ABC, abstractmethod
5
from collections import defaultdict
@@ -1156,6 +1155,18 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1155
1156
self.data_parser = self._get_data_parser()
1157
1158
+ # Avoid unnecessary recomputation
1159
+ self._supported_mm_limits = self.info.get_supported_mm_limits()
1160
+ self._allowed_mm_limits = self.info.get_allowed_mm_limits()
1161
+
1162
+ @property
1163
+ def supported_mm_limits(self):
1164
+ return self._supported_mm_limits
1165
+
1166
+ @property
1167
+ def allowed_mm_limits(self):
1168
+ return self._allowed_mm_limits
1169
+
1170
def __call__(
1171
self,
1172
prompt: str,
@@ -1176,6 +1187,28 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1187
"""
1188
return MultiModalDataParser()
1189
1190
+ def validate_num_items(
1191
+ self,
1192
+ modality: str,
1193
+ num_items: int,
1194
+ ) -> None:
1195
+ supported_limit = self.supported_mm_limits.get(modality, 0)
1196
+ allowed_limit = self.allowed_mm_limits.get(modality, 0)
1197
+
1198
+ if supported_limit is None:
1199
+ supported_limit = allowed_limit
1200
+
1201
+ limit = min(supported_limit, allowed_limit)
1202
+
1203
+ if num_items > limit:
1204
+ msg = (f"At most {limit} {modality}(s) may be provided in "
1205
+ "one prompt.")
1206
+
1207
+ if num_items <= supported_limit:
1208
+ msg += " Set `--limit-mm-per-prompt` to increase this limit."
1209
+
1210
+ raise ValueError(msg)
1211
+
1212
def _to_mm_items(
1213
self,
1214
mm_data: MultiModalDataDict,
@@ -1188,26 +1221,9 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1221
[`_get_hf_mm_data`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_data].
1222
"""
1223
mm_items = self.data_parser.parse_mm_data(mm_data)
1191
- supported_mm_limits = self.info.get_supported_mm_limits()
1192
- allowed_mm_limits = self.info.get_allowed_mm_limits()
1224
1225
for modality, items in mm_items.items():
1195
- supported_limit = supported_mm_limits.get(modality, 0)
1196
- allowed_limit = allowed_mm_limits.get(modality, 0)
1197
- num_items = len(items)
1198
-
1199
- if supported_limit is not None and num_items > supported_limit:
1200
- raise ValueError(
1201
- f"The model only supports at most {supported_limit} "
1202
- f"{modality} items, but you passed {num_items} "
1203
- f"{modality} items in the same prompt.")
1204
-
1205
- if num_items > allowed_limit:
1206
- raise ValueError(
1207
- "You set or defaulted to "
1208
- f"'{json.dumps({modality: allowed_limit})}' in "
1209
- f"`--limit-mm-per-prompt`, but passed {num_items} "
1210
- f"{modality} items in the same prompt.")
1226
+ self.validate_num_items(modality, len(items))
1227
1228
return mm_items
1229
vllm/multimodal/profiling.py
+1
-1
@@ -156,7 +156,7 @@ class MultiModalProfiler(Generic[_I]):
156
return self.processor.dummy_inputs
157
158
def get_mm_limits(self) -> Mapping[str, int]:
159
- return self.processing_info.get_allowed_mm_limits()
159
+ return self.processor.allowed_mm_limits
160
161
def _get_dummy_mm_inputs(
162
self,