@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 79f05e4436
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

[Multimodal] Always enable hashing mm data (#23308)

Signed-off-by: Roger Wang <hey@rogerw.io> Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>

Roger Wang committed Aug 21, 2025 at 07:23 UTC 79f05e4436fd97383bfd6319a1e80886bceb0fd3
15 files changed +95 -149
vllm/config/__init__.py
-9
@@ -1685,15 +1685,6 @@ class ModelConfig:
1685 def is_multimodal_model(self) -> bool:
1686 return self.multimodal_config is not None
1687
1688 - @property
1689 - def processor_return_mm_hashes(self) -> bool:
1690 - """Whether the multi-modal processor should output hashes."""
1691 - mm_config = self.multimodal_config
1692 - if mm_config is None:
1693 - return False
1694 -
1695 - return mm_config.mm_processor_cache_gb > 0
1696 -
1688 @property
1689 def enable_mm_processor_cache(self) -> bool:
1690 """Whether the multi-modal processor cache should be enabled."""
vllm/inputs/preprocess.py
+11 -42
@@ -254,7 +254,6 @@ class InputPreprocessor:
254 mm_processor_kwargs: Optional[Mapping[str, object]],
255 tokenization_kwargs: Optional[dict[str, Any]] = None,
256 lora_request: Optional[LoRARequest] = None,
257 - return_mm_hashes: bool = False,
257 ) -> MultiModalInputs:
258 """
259 Apply the model's multi-modal processor to a multi-modal prompt,
@@ -271,8 +270,7 @@ class InputPreprocessor:
270 return mm_processor.apply(prompt,
271 mm_data,
272 hf_processor_mm_kwargs=mm_processor_kwargs,
274 - tokenization_kwargs=tokenization_kwargs,
275 - return_mm_hashes=return_mm_hashes)
273 + tokenization_kwargs=tokenization_kwargs)
274
275 async def _process_multimodal_async(
276 self,
@@ -281,7 +279,6 @@ class InputPreprocessor:
279 mm_processor_kwargs: Optional[Mapping[str, object]],
280 tokenization_kwargs: Optional[dict[str, Any]] = None,
281 lora_request: Optional[LoRARequest] = None,
284 - return_mm_hashes: bool = False,
282 ) -> MultiModalInputs:
283 """
284 Async version of
@@ -297,8 +294,7 @@ class InputPreprocessor:
294 return mm_processor.apply(prompt,
295 mm_data,
296 hf_processor_mm_kwargs=mm_processor_kwargs,
300 - tokenization_kwargs=tokenization_kwargs,
301 - return_mm_hashes=return_mm_hashes)
297 + tokenization_kwargs=tokenization_kwargs)
298
299 def _process_embeds(
300 self,
@@ -335,7 +331,6 @@ class InputPreprocessor:
331 parsed_content: TokensPrompt,
332 tokenization_kwargs: Optional[dict[str, Any]] = None,
333 lora_request: Optional[LoRARequest] = None,
338 - return_mm_hashes: bool = False,
334 ) -> Union[TokenInputs, MultiModalInputs]:
335 prompt_token_ids = parsed_content["prompt_token_ids"]
336 token_type_ids = parsed_content.get("token_type_ids")
@@ -348,7 +343,6 @@ class InputPreprocessor:
343 parsed_content.get("mm_processor_kwargs"),
344 tokenization_kwargs=tokenization_kwargs,
345 lora_request=lora_request,
351 - return_mm_hashes=return_mm_hashes,
346 )
347 else:
348 inputs = token_inputs(
@@ -366,7 +360,6 @@ class InputPreprocessor:
360 parsed_content: TokensPrompt,
361 tokenization_kwargs: Optional[dict[str, Any]] = None,
362 lora_request: Optional[LoRARequest] = None,
369 - return_mm_hashes: bool = False,
363 ) -> Union[TokenInputs, MultiModalInputs]:
364 prompt_token_ids = parsed_content["prompt_token_ids"]
365 token_type_ids = parsed_content.get("token_type_ids")
@@ -379,7 +372,6 @@ class InputPreprocessor:
372 parsed_content.get("mm_processor_kwargs"),
373 tokenization_kwargs=tokenization_kwargs,
374 lora_request=lora_request,
382 - return_mm_hashes=return_mm_hashes,
375 )
376 else:
377 inputs = token_inputs(
@@ -397,7 +389,6 @@ class InputPreprocessor:
389 parsed_content: TextPrompt,
390 tokenization_kwargs: Optional[dict[str, Any]] = None,
391 lora_request: Optional[LoRARequest] = None,
400 - return_mm_hashes: bool = False,
392 ) -> Union[TokenInputs, MultiModalInputs]:
393 prompt_text = parsed_content["prompt"]
394
@@ -409,7 +400,6 @@ class InputPreprocessor:
400 parsed_content.get("mm_processor_kwargs"),
401 tokenization_kwargs=tokenization_kwargs,
402 lora_request=lora_request,
412 - return_mm_hashes=return_mm_hashes,
403 )
404 else:
405 prompt_token_ids = self._tokenize_prompt(
@@ -432,7 +422,6 @@ class InputPreprocessor:
422 parsed_content: TextPrompt,
423 tokenization_kwargs: Optional[dict[str, Any]] = None,
424 lora_request: Optional[LoRARequest] = None,
435 - return_mm_hashes: bool = False,
425 ) -> Union[TokenInputs, MultiModalInputs]:
426 prompt_text = parsed_content["prompt"]
427
@@ -444,7 +433,6 @@ class InputPreprocessor:
433 parsed_content.get("mm_processor_kwargs"),
434 tokenization_kwargs=tokenization_kwargs,
435 lora_request=lora_request,
447 - return_mm_hashes=return_mm_hashes,
436 )
437 else:
438 prompt_token_ids = await self._tokenize_prompt_async(
@@ -467,7 +455,6 @@ class InputPreprocessor:
455 prompt: SingletonPrompt,
456 tokenization_kwargs: Optional[dict[str, Any]] = None,
457 lora_request: Optional[LoRARequest] = None,
470 - return_mm_hashes: bool = False,
458 ) -> SingletonInputs:
459 """
460 Extract the singleton inputs from a prompt.
@@ -476,7 +463,6 @@ class InputPreprocessor:
463
464 * prompt: single encoder or decoder input prompt
465 * lora_request: this is only valid for decoder prompts
479 - * return_mm_hashes: whether to return multimodal hashes
466
467 Returns:
468
@@ -490,21 +476,18 @@ class InputPreprocessor:
476 return self._process_tokens(
477 parsed["content"],
478 lora_request=lora_request,
493 - return_mm_hashes=return_mm_hashes,
479 )
480 if parsed["type"] == "text":
481 return self._process_text(
482 parsed["content"],
483 tokenization_kwargs=tokenization_kwargs,
484 lora_request=lora_request,
500 - return_mm_hashes=return_mm_hashes,
485 )
486 if parsed["type"] == "str":
487 return self._process_text(
488 TextPrompt(prompt=parsed["content"]),
489 tokenization_kwargs=tokenization_kwargs,
490 lora_request=lora_request,
507 - return_mm_hashes=return_mm_hashes,
491 )
492
493 assert_never(parsed)
@@ -514,7 +497,6 @@ class InputPreprocessor:
497 prompt: SingletonPrompt,
498 tokenization_kwargs: Optional[dict[str, Any]] = None,
499 lora_request: Optional[LoRARequest] = None,
517 - return_mm_hashes: bool = False,
500 ) -> SingletonInputs:
501 """
502 Async version of
@@ -528,21 +510,18 @@ class InputPreprocessor:
510 return await self._process_tokens_async(
511 parsed["content"],
512 lora_request=lora_request,
531 - return_mm_hashes=return_mm_hashes,
513 )
514 if parsed["type"] == "text":
515 return await self._process_text_async(
516 parsed["content"],
517 tokenization_kwargs=tokenization_kwargs,
518 lora_request=lora_request,
538 - return_mm_hashes=return_mm_hashes,
519 )
520 if parsed["type"] == "str":
521 return await self._process_text_async(
522 TextPrompt(prompt=parsed["content"]),
523 tokenization_kwargs=tokenization_kwargs,
524 lora_request=lora_request,
545 - return_mm_hashes=return_mm_hashes,
525 )
526
527 assert_never(parsed)
@@ -785,7 +764,6 @@ class InputPreprocessor:
764 prompt: SingletonPrompt,
765 tokenization_kwargs: Optional[dict[str, Any]] = None,
766 lora_request: Optional[LoRARequest] = None,
788 - return_mm_hashes: bool = False,
767 ) -> DecoderOnlyInputs:
768 """
769 For decoder-only models:
@@ -796,7 +774,6 @@ class InputPreprocessor:
774
775 * prompt: input prompt
776 * lora_request
799 - * return_mm_hashes
777
778 Returns:
779
@@ -807,7 +784,6 @@ class InputPreprocessor:
784 prompt,
785 tokenization_kwargs=tokenization_kwargs,
786 lora_request=lora_request,
810 - return_mm_hashes=return_mm_hashes,
787 )
788
789 return self._build_decoder_only_llm_inputs(prompt_comps)
@@ -817,7 +793,6 @@ class InputPreprocessor:
793 prompt: SingletonPrompt,
794 tokenization_kwargs: Optional[dict[str, Any]] = None,
795 lora_request: Optional[LoRARequest] = None,
820 - return_mm_hashes: bool = False,
796 ) -> DecoderOnlyInputs:
797 """
798 Async version of
@@ -827,7 +802,6 @@ class InputPreprocessor:
802 prompt,
803 tokenization_kwargs=tokenization_kwargs,
804 lora_request=lora_request,
830 - return_mm_hashes=return_mm_hashes,
805 )
806
807 return self._build_decoder_only_llm_inputs(prompt_comps)
@@ -837,17 +811,15 @@ class InputPreprocessor:
811 prompt: PromptType,
812 tokenization_kwargs: Optional[dict[str, Any]] = None,
813 lora_request: Optional[LoRARequest] = None,
840 - return_mm_hashes: bool = False,
814 ) -> ProcessorInputs:
815 """Preprocess the input prompt."""
816 if self.model_config.is_encoder_decoder:
844 - assert not return_mm_hashes, (
845 - "Multimodal hashes for encoder-decoder models should not be ",
846 - "returned until they are supported on vLLM V1.")
817 # Encoder-decoder model requires special mapping of
848 - # input prompts to encoder & decoder
818 + # input prompts to encoder & decoder.
819 return self._process_encoder_decoder_prompt(
850 - prompt, tokenization_kwargs)
820 + prompt,
821 + tokenization_kwargs,
822 + )
823
824 if is_explicit_encoder_decoder_prompt(prompt):
825 raise ValueError("Cannot pass encoder-decoder prompt "
@@ -858,7 +830,6 @@ class InputPreprocessor:
830 prompt,
831 tokenization_kwargs=tokenization_kwargs,
832 lora_request=lora_request,
861 - return_mm_hashes=return_mm_hashes,
833 )
834
835 async def preprocess_async(
@@ -866,19 +837,18 @@ class InputPreprocessor:
837 prompt: PromptType,
838 tokenization_kwargs: Optional[dict[str, Any]] = None,
839 lora_request: Optional[LoRARequest] = None,
869 - return_mm_hashes: bool = False,
840 ) -> ProcessorInputs:
841 """
842 Async version of
843 [`preprocess`][vllm.inputs.preprocess.InputPreprocessor.preprocess].
844 """
845 if self.model_config.is_encoder_decoder:
876 - assert not return_mm_hashes, (
877 - "Multimodal hashes for encoder-decoder models should not be ",
878 - "returned until they are supported on vLLM V1.")
846 # Encoder-decoder model requires special mapping of
880 - # input prompts to encoder & decoder
881 - return await self._process_encoder_decoder_prompt_async(prompt)
847 + # input prompts to encoder & decoder.
848 + return await self._process_encoder_decoder_prompt_async(
849 + prompt,
850 + tokenization_kwargs,
851 + )
852
853 if is_explicit_encoder_decoder_prompt(prompt):
854 raise ValueError("Cannot pass encoder-decoder prompt "
@@ -889,5 +859,4 @@ class InputPreprocessor:
859 prompt,
860 tokenization_kwargs=tokenization_kwargs,
861 lora_request=lora_request,
892 - return_mm_hashes=return_mm_hashes,
862 )
vllm/model_executor/models/deepseek_vl2.py
-4
@@ -290,8 +290,6 @@ class DeepseekVL2MultiModalProcessor(
290 mm_data_items: MultiModalDataItems,
291 hf_processor_mm_kwargs: Mapping[str, object],
292 tokenization_kwargs: Mapping[str, object],
293 - *,
294 - return_mm_hashes: bool,
293 ) -> tuple[list[int], MultiModalProcessingInfo, bool]:
294 # The processor logic is different for len(images) <= 2 vs > 2
295 # Since the processing cache assumes that the processor output is
@@ -303,7 +301,6 @@ class DeepseekVL2MultiModalProcessor(
301 mm_data_items=mm_data_items,
302 hf_processor_mm_kwargs=hf_processor_mm_kwargs,
303 tokenization_kwargs=tokenization_kwargs,
306 - return_mm_hashes=return_mm_hashes,
304 )
305
306 return super()._cached_apply_hf_processor(
@@ -311,7 +308,6 @@ class DeepseekVL2MultiModalProcessor(
308 mm_data_items=mm_data_items,
309 hf_processor_mm_kwargs=hf_processor_mm_kwargs,
310 tokenization_kwargs=tokenization_kwargs,
314 - return_mm_hashes=return_mm_hashes,
311 )
312
313
vllm/model_executor/models/h2ovl.py
-4
@@ -479,8 +479,6 @@ class H2OVLMultiModalProcessor(
479 mm_data_items: MultiModalDataItems,
480 hf_processor_mm_kwargs: Mapping[str, object],
481 tokenization_kwargs: Mapping[str, object],
482 - *,
483 - return_mm_hashes: bool,
482 ) -> tuple[list[int], MultiModalProcessingInfo, bool]:
483 # The processor logic is different for len(images) <= 1 vs > 1
484 # Since the processing cache assumes that the processor output is
@@ -492,7 +490,6 @@ class H2OVLMultiModalProcessor(
490 mm_data_items=mm_data_items,
491 hf_processor_mm_kwargs=hf_processor_mm_kwargs,
492 tokenization_kwargs=tokenization_kwargs,
495 - return_mm_hashes=return_mm_hashes,
493 )
494
495 return super()._cached_apply_hf_processor(
@@ -500,7 +497,6 @@ class H2OVLMultiModalProcessor(
497 mm_data_items=mm_data_items,
498 hf_processor_mm_kwargs=hf_processor_mm_kwargs,
499 tokenization_kwargs=tokenization_kwargs,
503 - return_mm_hashes=return_mm_hashes,
500 )
501
502
vllm/model_executor/models/llava.py
+1 -2
@@ -795,7 +795,6 @@ class MantisMultiModalProcessor(LlavaMultiModalProcessor):
795 mm_data: MultiModalDataDict,
796 hf_processor_mm_kwargs: Mapping[str, object],
797 tokenization_kwargs: Optional[Mapping[str, object]] = None,
798 - return_mm_hashes: bool = False,
798 ) -> MultiModalInputs:
799 hf_config = self.info.get_hf_config()
800 image_token_id = hf_config.image_token_index
@@ -807,7 +806,7 @@ class MantisMultiModalProcessor(LlavaMultiModalProcessor):
806 )
807
808 result = super().apply(prompt, mm_data, hf_processor_mm_kwargs,
810 - tokenization_kwargs, return_mm_hashes)
809 + tokenization_kwargs)
810
811 mm_items = self._to_mm_items(mm_data)
812 mm_item_counts = mm_items.get_all_counts()
vllm/model_executor/models/mllama.py
+1 -2
@@ -168,10 +168,9 @@ class MllamaMultiModalProcessor(EncDecMultiModalProcessor[MllamaProcessingInfo]
168 mm_data: MultiModalDataDict,
169 hf_processor_mm_kwargs: Mapping[str, object],
170 tokenization_kwargs: Optional[Mapping[str, object]] = None,
171 - return_mm_hashes: bool = False,
171 ) -> MultiModalEncDecInputs:
172 mm_inputs = super().apply(prompt, mm_data, hf_processor_mm_kwargs,
174 - tokenization_kwargs, return_mm_hashes)
173 + tokenization_kwargs)
174
175 image_token_id = self.info.get_hf_config().image_token_index
176 # Check that the number of image tokens in the decoder prompt matches
vllm/model_executor/models/paligemma.py
+1 -2
@@ -194,10 +194,9 @@ class PaliGemmaMultiModalProcessor(
194 mm_data: MultiModalDataDict,
195 hf_processor_mm_kwargs: Mapping[str, object],
196 tokenization_kwargs: Optional[Mapping[str, object]] = None,
197 - return_mm_hashes: bool = False,
197 ) -> MultiModalInputs:
198 mm_inputs = super().apply(prompt, mm_data, hf_processor_mm_kwargs,
200 - tokenization_kwargs, return_mm_hashes)
199 + tokenization_kwargs)
200 prompt_token_ids = mm_inputs["prompt_token_ids"]
201
202 tokenizer = self.info.get_tokenizer()
vllm/model_executor/models/pixtral.py
-3
@@ -308,15 +308,12 @@ class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo]
308 mm_data_items: MultiModalDataItems,
309 hf_processor_mm_kwargs: Mapping[str, object],
310 tokenization_kwargs: Mapping[str, object],
311 - *,
312 - return_mm_hashes: bool,
311 ) -> tuple[list[int], MultiModalProcessingInfo, bool]:
312 prompt_ids, mm_info, _ = super()._cached_apply_hf_processor(
313 prompt=prompt,
314 mm_data_items=mm_data_items,
315 hf_processor_mm_kwargs=hf_processor_mm_kwargs,
316 tokenization_kwargs=tokenization_kwargs,
319 - return_mm_hashes=return_mm_hashes,
317 )
318
319 # NOTE: The tokens are already inserted by the chat template
vllm/model_executor/models/prithvi_geospatial_mae.py
+69 -45
@@ -18,7 +18,7 @@
18 """Inference-only IBM/NASA Prithvi Geospatial model."""
19
20 from collections.abc import Iterable, Mapping, Sequence
21 -from typing import Optional, Union
21 +from typing import Any, Optional, Union
22
23 import torch
24 import torch.nn as nn
@@ -32,18 +32,56 @@ from vllm.model_executor.models.interfaces import (
32 default_pooling_type)
33 from vllm.model_executor.models.utils import AutoWeightsLoader
34 from vllm.multimodal import MULTIMODAL_REGISTRY
35 -from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
36 - MultiModalFieldElem, MultiModalInputs,
37 - MultiModalKwargsItem,
38 - MultiModalKwargsItems,
39 - MultiModalSharedField, PlaceholderRange)
40 -from vllm.multimodal.parse import MultiModalDataItems
35 +from vllm.multimodal.inputs import (ImageItem, ModalityData,
36 + MultiModalDataDict, MultiModalFieldConfig,
37 + MultiModalInputs, MultiModalKwargsItems,
38 + PlaceholderRange)
39 +from vllm.multimodal.parse import (DictEmbeddingItems, ModalityDataItems,
40 + MultiModalDataItems, MultiModalDataParser)
41 from vllm.multimodal.processing import (BaseMultiModalProcessor,
42 BaseProcessingInfo, PromptUpdate)
43 from vllm.multimodal.profiling import BaseDummyInputsBuilder
44 from vllm.sequence import IntermediateTensors
45
46
47 +def _prithvi_field_config(hf_inputs: Mapping[str, torch.Tensor]):
48 + # This model receives in input a multi-dimensional tensor representing
49 + # a single image patch and therefore it is not to be split
50 + # into multiple elements, but rather to be considered a single one.
51 + # Hence, the decision of using a MultiModalSharedField.
52 + # The expected shape is (num_channels, width, height).
53 +
54 + # This model however allows the user to also submit multiple image
55 + # patches as a batch, adding a further dimension to the above shape.
56 + # At this stage we only support submitting one patch per request and
57 + # batching is achieved via vLLM batching.
58 + # TODO (christian-pinto): enable support for multi patch requests
59 + # in tandem with vLLM batching.
60 + return dict(
61 + pixel_values=MultiModalFieldConfig.shared(batch_size=1,
62 + modality="image"),
63 + location_coords=MultiModalFieldConfig.shared(batch_size=1,
64 + modality="image"),
65 + )
66 +
67 +
68 +class PrithviGeoSpatialMAEMultiModalDataParser(MultiModalDataParser):
69 +
70 + def _parse_image_data(
71 + self,
72 + data: Union[dict[str, torch.Tensor], ModalityData[ImageItem]],
73 + ) -> Optional[ModalityDataItems[Any, Any]]:
74 + if isinstance(data, dict):
75 + return DictEmbeddingItems(
76 + data,
77 + modality="image",
78 + required_fields={"pixel_values", "location_coords"},
79 + fields_factory=_prithvi_field_config,
80 + )
81 +
82 + return super()._parse_image_data(data)
83 +
84 +
85 class PrithviGeoSpatialMAEProcessingInfo(BaseProcessingInfo):
86
87 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
@@ -64,26 +102,26 @@ class PrithviGeoSpatialMAEInputBuilder(
102 # This model input is fixed and is in the form of a torch Tensor.
103 # The size of pixel_values might change in the cases where we resize
104 # the input but never exceeds the dimensions below.
67 - return {
105 + image_data = {
106 "pixel_values": torch.full((6, 512, 512), 1.0,
107 dtype=torch.float16),
108 "location_coords": torch.full((1, 2), 1.0, dtype=torch.float16),
109 }
110
111 + return {"image": image_data}
112 +
113
114 class PrithviGeoSpatialMAEMultiModalProcessor(BaseMultiModalProcessor):
115
116 + def _get_data_parser(self) -> MultiModalDataParser:
117 + return PrithviGeoSpatialMAEMultiModalDataParser()
118 +
119 def _get_mm_fields_config(
120 self,
121 hf_inputs: BatchFeature,
122 hf_processor_mm_kwargs: Mapping[str, object],
123 ) -> Mapping[str, MultiModalFieldConfig]:
81 - return dict(
82 - pixel_values=MultiModalFieldConfig.shared(batch_size=1,
83 - modality="image"),
84 - location_coords=MultiModalFieldConfig.shared(batch_size=1,
85 - modality="image"),
86 - )
124 + return _prithvi_field_config(hf_inputs)
125
126 def _get_prompt_updates(
127 self,
@@ -99,46 +137,32 @@ class PrithviGeoSpatialMAEMultiModalProcessor(BaseMultiModalProcessor):
137 mm_data: MultiModalDataDict,
138 hf_processor_mm_kwargs: Mapping[str, object],
139 tokenization_kwargs: Optional[Mapping[str, object]] = None,
102 - return_mm_hashes: bool = False,
140 ) -> MultiModalInputs:
104 - mm_kwargs = {}
141 + if "image" in mm_data:
142 + image_data = mm_data["image"]
143 + else:
144 + image_data = mm_data
145 + mm_data = {"image": mm_data}
146
106 - for k, v in mm_data.items():
107 - if isinstance(v, dict) and k == "image":
108 - mm_kwargs.update(v)
109 - else:
110 - mm_kwargs[k] = v
147 + mm_items = self._to_mm_items(mm_data)
148 + mm_hashes = self._hash_mm_items(mm_items, hf_processor_mm_kwargs,
149 + tokenization_kwargs or {})
150 mm_placeholders = {"image": [PlaceholderRange(offset=0, length=0)]}
151
113 - # This model receives in input a multi-dimensional tensor representing
114 - # a single image patch and therefore it is not to be split
115 - # into multiple elements, but rather to be considered a single one.
116 - # Hence, the decision of using a MultiModalSharedField.
117 - # The expected shape is (num_channels, width, height).
118 -
119 - # This model however allows the user to also submit multiple image
120 - # patches as a batch, adding a further dimension to the above shape.
121 - # At this stage we only support submitting one patch per request and
122 - # batching is achieved via vLLM batching.
123 - # TODO (christian-pinto): enable support for multi patch requests
124 - # in tandem with vLLM batching.
125 - multimodal_kwargs_items = [
126 - MultiModalKwargsItem.from_elems([
127 - MultiModalFieldElem(
128 - modality="image",
129 - key=key,
130 - data=data,
131 - field=MultiModalSharedField(1),
132 - ) for key, data in mm_kwargs.items()
133 - ])
134 - ]
152 + mm_processed_data = BatchFeature(image_data)
153 +
154 + mm_kwargs = MultiModalKwargsItems.from_hf_inputs(
155 + mm_processed_data,
156 + self._get_mm_fields_config(mm_processed_data,
157 + hf_processor_mm_kwargs),
158 + )
159
160 return MultiModalInputs(
161 type="multimodal",
162 prompt=prompt,
163 prompt_token_ids=[1],
140 - mm_kwargs=MultiModalKwargsItems.from_seq(multimodal_kwargs_items),
141 - mm_hashes=None,
164 + mm_kwargs=mm_kwargs,
165 + mm_hashes=mm_hashes,
166 mm_placeholders=mm_placeholders,
167 )
168
vllm/model_executor/models/transformers.py
-1
@@ -310,7 +310,6 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
310 mm_data: MultiModalDataDict,
311 hf_processor_mm_kwargs: Mapping[str, object],
312 tokenization_kwargs: Optional[Mapping[str, object]] = None,
313 - return_mm_hashes: bool = False,
313 ) -> MultiModalInputs:
314 """
315 Process multi-modal inputs to be used in vLLM.
vllm/model_executor/models/voxtral.py
-3
@@ -288,15 +288,12 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo]
288 mm_data_items: MultiModalDataItems,
289 hf_processor_mm_kwargs: Mapping[str, object],
290 tokenization_kwargs: Mapping[str, object],
291 - *,
292 - return_mm_hashes: bool,
291 ) -> tuple[list[int], MultiModalProcessingInfo, bool]:
292 prompt_ids, mm_info, _ = super()._cached_apply_hf_processor(
293 prompt=prompt,
294 mm_data_items=mm_data_items,
295 hf_processor_mm_kwargs=hf_processor_mm_kwargs,
296 tokenization_kwargs=tokenization_kwargs,
299 - return_mm_hashes=return_mm_hashes,
297 )
298
299 # NOTE: The tokens are already inserted by the chat template
vllm/multimodal/hasher.py
+1 -1
@@ -43,7 +43,7 @@ class MultiModalHasher:
43 return cls.item_to_bytes(
44 "image", np.asarray(convert_image_mode(obj, "RGBA")))
45 if isinstance(obj, torch.Tensor):
46 - return cls.item_to_bytes("tensor", obj.numpy())
46 + return cls.item_to_bytes("tensor", obj.cpu().numpy())
47 if isinstance(obj, np.ndarray):
48 # If the array is non-contiguous, we need to copy it first
49 arr_data = obj.data if obj.flags.c_contiguous else obj.tobytes()
vllm/multimodal/inputs.py
+1 -1
@@ -901,7 +901,7 @@ class MultiModalInputs(TypedDict):
901 mm_kwargs: MultiModalKwargsItems
902 """Keyword arguments to be directly passed to the model after batching."""
903
904 - mm_hashes: Optional["MultiModalHashDict"]
904 + mm_hashes: "MultiModalHashDict"
905 """The hashes of the multi-modal data."""
906
907 mm_placeholders: "MultiModalPlaceholderDict"
vllm/multimodal/processing.py
+4 -16
@@ -998,7 +998,7 @@ A collection of prompt updates with a similar structure as
998
999 class MultiModalProcessingInfo(NamedTuple):
1000 kwargs: MultiModalKwargsItems
1001 - hashes: Optional[MultiModalHashes]
1001 + hashes: MultiModalHashes
1002 prompt_updates: MultiModalPromptUpdates
1003
1004
@@ -1399,8 +1399,6 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1399 mm_data_items: MultiModalDataItems,
1400 hf_processor_mm_kwargs: Mapping[str, object],
1401 tokenization_kwargs: Mapping[str, object],
1402 - *,
1403 - return_mm_hashes: bool,
1402 ) -> tuple[list[int], MultiModalProcessingInfo, bool]:
1403 (
1404 prompt_ids,
@@ -1420,9 +1418,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1418 hf_processor_mm_kwargs),
1419 )
1420
1423 - mm_hashes = (self._hash_mm_items(mm_data_items, hf_processor_mm_kwargs,
1424 - tokenization_kwargs)
1425 - if return_mm_hashes else None)
1421 + mm_hashes = self._hash_mm_items(mm_data_items, hf_processor_mm_kwargs,
1422 + tokenization_kwargs)
1423
1424 unbound_prompt_updates = self._get_prompt_updates(
1425 mm_data_items,
@@ -1446,8 +1443,6 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1443 mm_data_items: MultiModalDataItems,
1444 hf_processor_mm_kwargs: Mapping[str, object],
1445 tokenization_kwargs: Mapping[str, object],
1449 - *,
1450 - return_mm_hashes: bool,
1446 ) -> tuple[list[int], MultiModalProcessingInfo, bool]:
1447 """
1448 Apply the HF processor on the full prompt text,
@@ -1462,7 +1457,6 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1457 mm_data_items=mm_data_items,
1458 hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1459 tokenization_kwargs=tokenization_kwargs,
1465 - return_mm_hashes=return_mm_hashes,
1460 )
1461
1462 mm_hashes = self._hash_mm_items(mm_data_items, hf_processor_mm_kwargs,
@@ -1476,8 +1470,6 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1470 mm_hashes=mm_hashes,
1471 )
1472
1479 - mm_hashes_to_return = mm_hashes if return_mm_hashes else None
1480 -
1473 # NOTE: `prompt` does not correspond to `mm_missing_data_items`,
1474 # so we can't apply prompt updates until the new multimodal
1475 # items are combined with the cached multimodal items
@@ -1515,7 +1507,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1507
1508 mm_info = MultiModalProcessingInfo(
1509 kwargs=mm_kwargs,
1518 - hashes=mm_hashes_to_return,
1510 + hashes=mm_hashes,
1511 prompt_updates=mm_prompt_updates,
1512 )
1513
@@ -1697,7 +1689,6 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1689 mm_data: MultiModalDataDict,
1690 hf_processor_mm_kwargs: Mapping[str, object],
1691 tokenization_kwargs: Optional[Mapping[str, object]] = None,
1700 - return_mm_hashes: bool = False,
1692 ) -> MultiModalInputs:
1693 """
1694 Process multi-modal inputs to be used in vLLM.
@@ -1726,7 +1717,6 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1717 mm_items,
1718 hf_processor_mm_kwargs,
1719 tokenization_kwargs=tokenization_kwargs,
1729 - return_mm_hashes=return_mm_hashes,
1720 )
1721
1722 # NOTE: tokenization_kwargs are not required to init processor
@@ -1811,7 +1801,6 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
1801 mm_data: MultiModalDataDict,
1802 hf_processor_mm_kwargs: Mapping[str, object],
1803 tokenization_kwargs: Optional[Mapping[str, object]] = None,
1814 - return_mm_hashes: bool = False,
1804 ) -> MultiModalEncDecInputs:
1805 """
1806 Process multi-modal inputs to be used in vLLM.
@@ -1826,7 +1815,6 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
1815 mm_data,
1816 hf_processor_mm_kwargs,
1817 tokenization_kwargs,
1829 - return_mm_hashes,
1818 )
1819
1820 return self._get_enc_dec_inputs(
vllm/v1/engine/processor.py
+6 -14
@@ -17,7 +17,6 @@ from vllm.multimodal.utils import argsort_mm_positions
17 from vllm.pooling_params import PoolingParams
18 from vllm.sampling_params import SamplingParams
19 from vllm.transformers_utils.tokenizer_group import TokenizerGroup
20 -from vllm.utils import is_list_of
20 from vllm.v1.engine import EngineCoreRequest
21 from vllm.v1.engine.mm_input_cache import MultiModalInputCacheClient
22 from vllm.v1.structured_output.backend_guidance import (
@@ -253,13 +252,10 @@ class Processor:
252 # 1. Tokenize text prompt, with LoRA request if one exists.
253 # 2. For multimodal models with a merged preprocessor, preprocess
254 # multimodal data and expand prompt token ids accordingly.
256 - return_mm_hashes = (self.model_config.processor_return_mm_hashes
257 - or bool(self.cache_config.enable_prefix_caching))
255 processed_inputs: ProcessorInputs = self.input_preprocessor.preprocess(
256 prompt,
257 tokenization_kwargs=tokenization_kwargs,
258 lora_request=lora_request,
262 - return_mm_hashes=return_mm_hashes,
259 )
260 from vllm.platforms import current_platform
261 current_platform.validate_request(
@@ -302,7 +298,7 @@ class Processor:
298 if decoder_inputs["type"] == "multimodal":
299 decoder_mm_inputs = decoder_inputs["mm_kwargs"]
300 decoder_mm_positions = decoder_inputs["mm_placeholders"]
305 - decoder_mm_hashes = decoder_inputs.get("mm_hashes")
301 + decoder_mm_hashes = decoder_inputs["mm_hashes"]
302
303 # Merge and flatten multimodal placeholders, hashes and inputs
304 # from dictionaries to lists, and sort them by each item's position
@@ -317,19 +313,15 @@ class Processor:
313 decoder_mm_positions[modality][idx]
314 for modality, idx in sorted_mm_idxs
315 ]
320 - sorted_mm_hashes = None if decoder_mm_hashes is None else [
316 + sorted_mm_hashes = [
317 decoder_mm_hashes[modality][idx]
318 for modality, idx in sorted_mm_idxs
319 ]
320
325 - if sorted_mm_hashes is not None:
326 - sorted_mm_inputs = self.mm_input_cache_client.get_and_update(
327 - orig_sorted_mm_inputs,
328 - sorted_mm_hashes,
329 - )
330 - else:
331 - assert is_list_of(orig_sorted_mm_inputs, MultiModalKwargsItem)
332 - sorted_mm_inputs = orig_sorted_mm_inputs
321 + sorted_mm_inputs = self.mm_input_cache_client.get_and_update(
322 + orig_sorted_mm_inputs,
323 + sorted_mm_hashes,
324 + )
325
326 return decoder_inputs.get("prompt"), EngineCoreRequest(
327 request_id=request_id,