A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 21:00 UTC
[Bugfix] Check that number of images matches number of <|image|> tokens with mllama (#13911)
Signed-off-by: Travis Johnson <tsjohnso@us.ibm.com>
Travis Johnson committed
Feb 27, 2025 at 21:00 UTC
73e0225ee9defc6fa57db3ac53d178b8a3fc169b
2 files changed
+26
-3
tests/models/encoder_decoder/vision_language/test_mllama.py
+3
-2
@@ -479,8 +479,9 @@ def test_regression(vllm_runner, image_assets, model, dtype, max_tokens,
479
480
# Regression tests for https://github.com/vllm-project/vllm/issues/10648
481
482
- # Number of image tags is greater than the number of images provided
483
- prompt = "<|begin_of_text|><|image|><|image|> Compare the two images" # noqa: E501
482
+ # Number of groups of image tokens is greater than the number of images
483
+ # provided (the whitespace between the tags is necessary)
484
+ prompt = "<|begin_of_text|><|image|> <|image|> Compare the two images" # noqa: E501
485
image = stop_sign
486
with pytest.raises(ValueError):
487
vllm_model.generate_greedy_logprobs([prompt],
vllm/model_executor/models/mllama.py
+23
-1
@@ -54,7 +54,8 @@ from vllm.model_executor.model_loader.weight_utils import (
54
default_weight_loader, maybe_remap_kv_scale_name)
55
from vllm.model_executor.sampling_metadata import SamplingMetadata
56
from vllm.multimodal import MULTIMODAL_REGISTRY
57
-from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargs
57
+from vllm.multimodal.inputs import (MultiModalEncDecInputs,
58
+ MultiModalFieldConfig, MultiModalKwargs)
59
from vllm.multimodal.parse import (ImageProcessorItems, ImageSize,
60
MultiModalDataDict, MultiModalDataItems)
61
from vllm.multimodal.processing import (BaseProcessingInfo,
@@ -169,6 +170,27 @@ class MllamaDummyInputsBuilder(BaseDummyInputsBuilder[MllamaProcessingInfo]):
170
class MllamaMultiModalProcessor(EncDecMultiModalProcessor[MllamaProcessingInfo]
171
):
172
173
+ def apply(
174
+ self,
175
+ prompt: Union[str, list[int]],
176
+ mm_data: MultiModalDataDict,
177
+ hf_processor_mm_kwargs: Mapping[str, object],
178
+ ) -> MultiModalEncDecInputs:
179
+ mm_inputs = super().apply(prompt, mm_data, hf_processor_mm_kwargs)
180
+
181
+ # Check that the number of image tokens in the decoder prompt matches
182
+ # the number of images provided in mm_data
183
+ num_image_tokens = mm_inputs['prompt_token_ids'].count(
184
+ self.info.get_hf_config().image_token_index)
185
+ image_data = mm_data.get("image", [])
186
+ num_images = 1 if isinstance(image_data, Image) else len(image_data)
187
+ if num_image_tokens != num_images:
188
+ raise ValueError(
189
+ f"The number of image tokens ({num_image_tokens}) must be"
190
+ f" the same as the number of images ({num_images})")
191
+
192
+ return mm_inputs
193
+
194
def _call_hf_processor(
195
self,
196
prompt: str,