A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Core] Simplify and unify mm uuid handling & auto-generated mm hash overrides processing. (#24271)
Signed-off-by: Chenheli Hua <huachenheli@outlook.com>
Chenheli Hua committed
Sep 9, 2025 at 21:36 UTC
009d689b0c30bc16301e39ca258219ecfbd77f20
13 files changed
+110
-127
tests/v1/engine/test_processor_multi_modal_uuids.py
+6
-6
@@ -152,8 +152,8 @@ def test_multi_modal_uuids_accepts_none_and_passes_through(
152
*,
153
tokenization_kwargs=None,
154
lora_request=None,
155
- mm_hash_overrides=None):
156
- captured["mm_hash_overrides"] = mm_hash_overrides
155
+ mm_uuids=None):
156
+ captured["mm_uuids"] = mm_uuids
157
# Minimal processed inputs for decoder-only flow
158
return {"type": "token", "prompt_token_ids": [1]}
159
@@ -180,7 +180,7 @@ def test_multi_modal_uuids_accepts_none_and_passes_through(
180
params=SamplingParams(),
181
)
182
183
- assert captured["mm_hash_overrides"] == mm_uuids
183
+ assert captured["mm_uuids"] == mm_uuids
184
185
186
def test_multi_modal_uuids_ignored_when_caching_disabled(monkeypatch):
@@ -196,8 +196,8 @@ def test_multi_modal_uuids_ignored_when_caching_disabled(monkeypatch):
196
*,
197
tokenization_kwargs=None,
198
lora_request=None,
199
- mm_hash_overrides=None):
200
- captured["mm_hash_overrides"] = mm_hash_overrides
199
+ mm_uuids=None):
200
+ captured["mm_uuids"] = mm_uuids
201
return {"type": "token", "prompt_token_ids": [1]}
202
203
monkeypatch.setattr(processor.input_preprocessor,
@@ -223,7 +223,7 @@ def test_multi_modal_uuids_ignored_when_caching_disabled(monkeypatch):
223
)
224
225
# Expect request-id-based overrides are passed through
226
- assert captured["mm_hash_overrides"] == {
226
+ assert captured["mm_uuids"] == {
227
"image": [f"{request_id}-image-0", f"{request_id}-image-1"],
228
"video": [f"{request_id}-video-0"],
229
}
vllm/inputs/preprocess.py
+37
-51
@@ -258,8 +258,7 @@ class InputPreprocessor:
258
tokenization_kwargs: Optional[dict[str, Any]] = None,
259
lora_request: Optional[LoRARequest] = None,
260
*,
261
- mm_hash_overrides: Optional[Union[dict[str, list[str]],
262
- MultiModalUUIDDict]] = None,
261
+ mm_uuids: Optional[MultiModalUUIDDict] = None,
262
) -> MultiModalInputs:
263
"""
264
Apply the model's multi-modal processor to a multi-modal prompt,
@@ -281,7 +280,7 @@ class InputPreprocessor:
280
mm_data,
281
hf_processor_mm_kwargs=mm_processor_kwargs,
282
tokenization_kwargs=tokenization_kwargs,
284
- mm_hash_overrides=mm_hash_overrides,
283
+ mm_uuids=mm_uuids,
284
)
285
mm_hashes = mm_input["mm_hashes"]
286
@@ -302,8 +301,7 @@ class InputPreprocessor:
301
tokenization_kwargs: Optional[dict[str, Any]] = None,
302
lora_request: Optional[LoRARequest] = None,
303
*,
305
- mm_hash_overrides: Optional[Union[dict[str, list[str]],
306
- MultiModalUUIDDict]] = None,
304
+ mm_uuids: Optional[MultiModalUUIDDict] = None,
305
) -> MultiModalInputs:
306
"""
307
Async version of
@@ -325,7 +323,7 @@ class InputPreprocessor:
323
mm_data,
324
hf_processor_mm_kwargs=mm_processor_kwargs,
325
tokenization_kwargs=tokenization_kwargs,
328
- mm_hash_overrides=mm_hash_overrides,
326
+ mm_uuids=mm_uuids,
327
)
328
mm_hashes = mm_input["mm_hashes"]
329
@@ -390,8 +388,7 @@ class InputPreprocessor:
388
tokenization_kwargs: Optional[dict[str, Any]] = None,
389
lora_request: Optional[LoRARequest] = None,
390
*,
393
- mm_hash_overrides: Optional[Union[dict[str, list[str]],
394
- MultiModalUUIDDict]] = None,
391
+ mm_uuids: Optional[MultiModalUUIDDict] = None,
392
) -> Union[TokenInputs, MultiModalInputs]:
393
prompt_token_ids = self._truncate_inputs(
394
parsed_content["prompt_token_ids"], tokenization_kwargs)
@@ -404,7 +401,7 @@ class InputPreprocessor:
401
parsed_content.get("mm_processor_kwargs"),
402
tokenization_kwargs=tokenization_kwargs,
403
lora_request=lora_request,
407
- mm_hash_overrides=mm_hash_overrides,
404
+ mm_uuids=mm_uuids,
405
)
406
else:
407
inputs = token_inputs(prompt_token_ids=prompt_token_ids)
@@ -420,8 +417,7 @@ class InputPreprocessor:
417
tokenization_kwargs: Optional[dict[str, Any]] = None,
418
lora_request: Optional[LoRARequest] = None,
419
*,
423
- mm_hash_overrides: Optional[Union[dict[str, list[str]],
424
- MultiModalUUIDDict]] = None,
420
+ mm_uuids: Optional[MultiModalUUIDDict] = None,
421
) -> Union[TokenInputs, MultiModalInputs]:
422
prompt_token_ids = self._truncate_inputs(
423
parsed_content["prompt_token_ids"], tokenization_kwargs)
@@ -434,7 +430,7 @@ class InputPreprocessor:
430
parsed_content.get("mm_processor_kwargs"),
431
tokenization_kwargs=tokenization_kwargs,
432
lora_request=lora_request,
437
- mm_hash_overrides=mm_hash_overrides,
433
+ mm_uuids=mm_uuids,
434
)
435
else:
436
inputs = token_inputs(prompt_token_ids=prompt_token_ids, )
@@ -450,8 +446,7 @@ class InputPreprocessor:
446
tokenization_kwargs: Optional[dict[str, Any]] = None,
447
lora_request: Optional[LoRARequest] = None,
448
*,
453
- mm_hash_overrides: Optional[Union[dict[str, list[str]],
454
- MultiModalUUIDDict]] = None,
449
+ mm_uuids: Optional[MultiModalUUIDDict] = None,
450
) -> Union[TokenInputs, MultiModalInputs]:
451
prompt_text = parsed_content["prompt"]
452
@@ -463,7 +458,7 @@ class InputPreprocessor:
458
parsed_content.get("mm_processor_kwargs"),
459
tokenization_kwargs=tokenization_kwargs,
460
lora_request=lora_request,
466
- mm_hash_overrides=mm_hash_overrides,
461
+ mm_uuids=mm_uuids,
462
)
463
else:
464
prompt_token_ids = self._tokenize_prompt(
@@ -487,8 +482,7 @@ class InputPreprocessor:
482
tokenization_kwargs: Optional[dict[str, Any]] = None,
483
lora_request: Optional[LoRARequest] = None,
484
*,
490
- mm_hash_overrides: Optional[Union[dict[str, list[str]],
491
- MultiModalUUIDDict]] = None,
485
+ mm_uuids: Optional[MultiModalUUIDDict] = None,
486
) -> Union[TokenInputs, MultiModalInputs]:
487
prompt_text = parsed_content["prompt"]
488
@@ -500,7 +494,7 @@ class InputPreprocessor:
494
parsed_content.get("mm_processor_kwargs"),
495
tokenization_kwargs=tokenization_kwargs,
496
lora_request=lora_request,
503
- mm_hash_overrides=mm_hash_overrides,
497
+ mm_uuids=mm_uuids,
498
)
499
else:
500
prompt_token_ids = await self._tokenize_prompt_async(
@@ -524,8 +518,7 @@ class InputPreprocessor:
518
tokenization_kwargs: Optional[dict[str, Any]] = None,
519
lora_request: Optional[LoRARequest] = None,
520
*,
527
- mm_hash_overrides: Optional[Union[dict[str, list[str]],
528
- MultiModalUUIDDict]] = None,
521
+ mm_uuids: Optional[MultiModalUUIDDict] = None,
522
) -> SingletonInputs:
523
"""
524
Extract the singleton inputs from a prompt.
@@ -547,21 +540,21 @@ class InputPreprocessor:
540
return self._process_tokens(
541
parsed["content"],
542
lora_request=lora_request,
550
- mm_hash_overrides=mm_hash_overrides,
543
+ mm_uuids=mm_uuids,
544
)
545
if parsed["type"] == "text":
546
return self._process_text(
547
parsed["content"],
548
tokenization_kwargs=tokenization_kwargs,
549
lora_request=lora_request,
557
- mm_hash_overrides=mm_hash_overrides,
550
+ mm_uuids=mm_uuids,
551
)
552
if parsed["type"] == "str":
553
return self._process_text(
554
TextPrompt(prompt=parsed["content"]),
555
tokenization_kwargs=tokenization_kwargs,
556
lora_request=lora_request,
564
- mm_hash_overrides=mm_hash_overrides,
557
+ mm_uuids=mm_uuids,
558
)
559
560
assert_never(parsed)
@@ -572,8 +565,7 @@ class InputPreprocessor:
565
tokenization_kwargs: Optional[dict[str, Any]] = None,
566
lora_request: Optional[LoRARequest] = None,
567
*,
575
- mm_hash_overrides: Optional[Union[dict[str, list[str]],
576
- MultiModalUUIDDict]] = None,
568
+ mm_uuids: Optional[MultiModalUUIDDict] = None,
569
) -> SingletonInputs:
570
"""
571
Async version of
@@ -587,21 +579,21 @@ class InputPreprocessor:
579
return await self._process_tokens_async(
580
parsed["content"],
581
lora_request=lora_request,
590
- mm_hash_overrides=mm_hash_overrides,
582
+ mm_uuids=mm_uuids,
583
)
584
if parsed["type"] == "text":
585
return await self._process_text_async(
586
parsed["content"],
587
tokenization_kwargs=tokenization_kwargs,
588
lora_request=lora_request,
597
- mm_hash_overrides=mm_hash_overrides,
589
+ mm_uuids=mm_uuids,
590
)
591
if parsed["type"] == "str":
592
return await self._process_text_async(
593
TextPrompt(prompt=parsed["content"]),
594
tokenization_kwargs=tokenization_kwargs,
595
lora_request=lora_request,
604
- mm_hash_overrides=mm_hash_overrides,
596
+ mm_uuids=mm_uuids,
597
)
598
599
assert_never(parsed)
@@ -712,8 +704,7 @@ class InputPreprocessor:
704
prompt: PromptType,
705
tokenization_kwargs: Optional[dict[str, Any]] = None,
706
*,
715
- mm_hash_overrides: Optional[Union[dict[str, list[str]],
716
- MultiModalUUIDDict]] = None,
707
+ mm_uuids: Optional[MultiModalUUIDDict] = None,
708
) -> EncoderDecoderInputs:
709
"""
710
For encoder/decoder models only:
@@ -755,7 +746,7 @@ class InputPreprocessor:
746
encoder_inputs = self._prompt_to_llm_inputs(
747
prompt["encoder_prompt"],
748
tokenization_kwargs=tokenization_kwargs,
758
- mm_hash_overrides=mm_hash_overrides,
749
+ mm_uuids=mm_uuids,
750
)
751
if (decoder_input := prompt["decoder_prompt"]) is None:
752
decoder_inputs = None
@@ -771,7 +762,7 @@ class InputPreprocessor:
762
inputs = self._prompt_to_llm_inputs(
763
prompt,
764
tokenization_kwargs=tokenization_kwargs,
774
- mm_hash_overrides=mm_hash_overrides,
765
+ mm_uuids=mm_uuids,
766
)
767
if self.model_config.is_multimodal_model:
768
# Encoder-Decoder Multimodal model
@@ -788,8 +779,7 @@ class InputPreprocessor:
779
prompt: PromptType,
780
tokenization_kwargs: Optional[dict[str, Any]] = None,
781
*,
791
- mm_hash_overrides: Optional[Union[dict[str, list[str]],
792
- MultiModalUUIDDict]] = None,
782
+ mm_uuids: Optional[MultiModalUUIDDict] = None,
783
) -> EncoderDecoderInputs:
784
"""
785
Async version of
@@ -802,7 +792,7 @@ class InputPreprocessor:
792
encoder_task = self._prompt_to_llm_inputs_async(
793
prompt["encoder_prompt"],
794
tokenization_kwargs=tokenization_kwargs,
805
- mm_hash_overrides=mm_hash_overrides,
795
+ mm_uuids=mm_uuids,
796
)
797
798
if (decoder_input := prompt["decoder_prompt"]) is None:
@@ -812,7 +802,7 @@ class InputPreprocessor:
802
decoder_task = self._prompt_to_llm_inputs_async(
803
decoder_input,
804
tokenization_kwargs=tokenization_kwargs,
815
- mm_hash_overrides=mm_hash_overrides,
805
+ mm_uuids=mm_uuids,
806
)
807
808
encoder_inputs, decoder_inputs = await asyncio.gather(
@@ -828,7 +818,7 @@ class InputPreprocessor:
818
inputs = await self._prompt_to_llm_inputs_async(
819
prompt,
820
tokenization_kwargs=tokenization_kwargs,
831
- mm_hash_overrides=mm_hash_overrides,
821
+ mm_uuids=mm_uuids,
822
)
823
if self.model_config.is_multimodal_model:
824
# Encoder-Decoder Multimodal model
@@ -856,8 +846,7 @@ class InputPreprocessor:
846
tokenization_kwargs: Optional[dict[str, Any]] = None,
847
lora_request: Optional[LoRARequest] = None,
848
*,
859
- mm_hash_overrides: Optional[Union[dict[str, list[str]],
860
- MultiModalUUIDDict]] = None,
849
+ mm_uuids: Optional[MultiModalUUIDDict] = None,
850
) -> DecoderOnlyInputs:
851
"""
852
For decoder-only models:
@@ -878,7 +867,7 @@ class InputPreprocessor:
867
prompt,
868
tokenization_kwargs=tokenization_kwargs,
869
lora_request=lora_request,
881
- mm_hash_overrides=mm_hash_overrides,
870
+ mm_uuids=mm_uuids,
871
)
872
873
return self._build_decoder_only_llm_inputs(prompt_comps)
@@ -889,8 +878,7 @@ class InputPreprocessor:
878
tokenization_kwargs: Optional[dict[str, Any]] = None,
879
lora_request: Optional[LoRARequest] = None,
880
*,
892
- mm_hash_overrides: Optional[Union[dict[str, list[str]],
893
- MultiModalUUIDDict]] = None,
881
+ mm_uuids: Optional[MultiModalUUIDDict] = None,
882
) -> DecoderOnlyInputs:
883
"""
884
Async version of
@@ -900,7 +888,7 @@ class InputPreprocessor:
888
prompt,
889
tokenization_kwargs=tokenization_kwargs,
890
lora_request=lora_request,
903
- mm_hash_overrides=mm_hash_overrides,
891
+ mm_uuids=mm_uuids,
892
)
893
894
return self._build_decoder_only_llm_inputs(prompt_comps)
@@ -911,8 +899,7 @@ class InputPreprocessor:
899
tokenization_kwargs: Optional[dict[str, Any]] = None,
900
lora_request: Optional[LoRARequest] = None,
901
*,
914
- mm_hash_overrides: Optional[Union[dict[str, list[str]],
915
- MultiModalUUIDDict]] = None,
902
+ mm_uuids: Optional[MultiModalUUIDDict] = None,
903
) -> ProcessorInputs:
904
"""Preprocess the input prompt."""
905
if self.model_config.is_encoder_decoder:
@@ -921,7 +908,7 @@ class InputPreprocessor:
908
return self._process_encoder_decoder_prompt(
909
prompt,
910
tokenization_kwargs,
924
- mm_hash_overrides=mm_hash_overrides,
911
+ mm_uuids=mm_uuids,
912
)
913
914
if is_explicit_encoder_decoder_prompt(prompt):
@@ -933,7 +920,7 @@ class InputPreprocessor:
920
prompt,
921
tokenization_kwargs=tokenization_kwargs,
922
lora_request=lora_request,
936
- mm_hash_overrides=mm_hash_overrides,
923
+ mm_uuids=mm_uuids,
924
)
925
926
async def preprocess_async(
@@ -942,8 +929,7 @@ class InputPreprocessor:
929
tokenization_kwargs: Optional[dict[str, Any]] = None,
930
lora_request: Optional[LoRARequest] = None,
931
*,
945
- mm_hash_overrides: Optional[Union[dict[str, list[str]],
946
- MultiModalUUIDDict]] = None,
932
+ mm_uuids: Optional[MultiModalUUIDDict] = None,
933
) -> ProcessorInputs:
934
"""
935
Async version of
@@ -955,7 +941,7 @@ class InputPreprocessor:
941
return await self._process_encoder_decoder_prompt_async(
942
prompt,
943
tokenization_kwargs,
958
- mm_hash_overrides=mm_hash_overrides,
944
+ mm_uuids=mm_uuids,
945
)
946
947
if is_explicit_encoder_decoder_prompt(prompt):
@@ -967,7 +953,7 @@ class InputPreprocessor:
953
prompt,
954
tokenization_kwargs=tokenization_kwargs,
955
lora_request=lora_request,
970
- mm_hash_overrides=mm_hash_overrides,
956
+ mm_uuids=mm_uuids,
957
)
958
959
def clear_cache(self) -> None:
vllm/model_executor/models/deepseek_vl2.py
+5
-4
@@ -21,7 +21,8 @@ from vllm.model_executor.model_loader.utils import set_default_torch_dtype
21
from vllm.model_executor.models.transformers import replace_linear_class
22
from vllm.multimodal import MULTIMODAL_REGISTRY
23
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
24
- MultiModalKwargsItems, NestedTensors)
24
+ MultiModalKwargsItems, MultiModalUUIDDict,
25
+ NestedTensors)
26
from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
27
ImageSize, MultiModalDataItems)
28
from vllm.multimodal.processing import (BaseMultiModalProcessor,
@@ -290,7 +291,7 @@ class DeepseekVL2MultiModalProcessor(
291
mm_data_items: MultiModalDataItems,
292
hf_processor_mm_kwargs: Mapping[str, object],
293
tokenization_kwargs: Mapping[str, object],
293
- mm_hash_overrides: Optional[dict[str, list[str]]] = None,
294
+ mm_uuids: Optional[MultiModalUUIDDict] = None,
295
) -> tuple[list[int], MultiModalProcessingInfo, bool]:
296
# The processor logic is different for len(images) <= 2 vs > 2
297
# Since the processing cache assumes that the processor output is
@@ -302,7 +303,7 @@ class DeepseekVL2MultiModalProcessor(
303
mm_data_items=mm_data_items,
304
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
305
tokenization_kwargs=tokenization_kwargs,
305
- mm_hash_overrides=mm_hash_overrides,
306
+ mm_uuids=mm_uuids,
307
)
308
309
return super()._cached_apply_hf_processor(
@@ -310,7 +311,7 @@ class DeepseekVL2MultiModalProcessor(
311
mm_data_items=mm_data_items,
312
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
313
tokenization_kwargs=tokenization_kwargs,
313
- mm_hash_overrides=mm_hash_overrides,
314
+ mm_uuids=mm_uuids,
315
)
316
317
vllm/model_executor/models/h2ovl.py
+4
-4
@@ -17,7 +17,7 @@ from transformers import PretrainedConfig
17
18
from vllm.model_executor.layers.quantization import QuantizationConfig
19
from vllm.multimodal import MULTIMODAL_REGISTRY
20
-from vllm.multimodal.inputs import MultiModalKwargsItems
20
+from vllm.multimodal.inputs import MultiModalKwargsItems, MultiModalUUIDDict
21
from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
22
MultiModalDataItems)
23
from vllm.multimodal.processing import (MultiModalProcessingInfo,
@@ -479,7 +479,7 @@ class H2OVLMultiModalProcessor(
479
mm_data_items: MultiModalDataItems,
480
hf_processor_mm_kwargs: Mapping[str, object],
481
tokenization_kwargs: Mapping[str, object],
482
- mm_hash_overrides: Optional[dict[str, list[str]]] = None,
482
+ mm_uuids: Optional[MultiModalUUIDDict] = None,
483
) -> tuple[list[int], MultiModalProcessingInfo, bool]:
484
# The processor logic is different for len(images) <= 1 vs > 1
485
# Since the processing cache assumes that the processor output is
@@ -491,7 +491,7 @@ class H2OVLMultiModalProcessor(
491
mm_data_items=mm_data_items,
492
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
493
tokenization_kwargs=tokenization_kwargs,
494
- mm_hash_overrides=mm_hash_overrides,
494
+ mm_uuids=mm_uuids,
495
)
496
497
return super()._cached_apply_hf_processor(
@@ -499,7 +499,7 @@ class H2OVLMultiModalProcessor(
499
mm_data_items=mm_data_items,
500
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
501
tokenization_kwargs=tokenization_kwargs,
502
- mm_hash_overrides=mm_hash_overrides,
502
+ mm_uuids=mm_uuids,
503
)
504
505
vllm/model_executor/models/llava.py
+4
-3
@@ -24,7 +24,8 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
24
from vllm.multimodal import MULTIMODAL_REGISTRY
25
from vllm.multimodal.cache import BaseMultiModalProcessorCache
26
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
27
- MultiModalInputs, MultiModalKwargsItems)
27
+ MultiModalInputs, MultiModalKwargsItems,
28
+ MultiModalUUIDDict)
29
from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
30
ImageSize, MultiModalDataItems)
31
from vllm.multimodal.processing import (BaseMultiModalProcessor,
@@ -795,7 +796,7 @@ class MantisMultiModalProcessor(LlavaMultiModalProcessor):
796
mm_data: MultiModalDataDict,
797
hf_processor_mm_kwargs: Mapping[str, object],
798
tokenization_kwargs: Optional[Mapping[str, object]] = None,
798
- mm_hash_overrides: Optional[dict[str, list[str]]] = None,
799
+ mm_uuids: Optional[MultiModalUUIDDict] = None,
800
) -> MultiModalInputs:
801
hf_config = self.info.get_hf_config()
802
image_token_id = hf_config.image_token_index
@@ -810,7 +811,7 @@ class MantisMultiModalProcessor(LlavaMultiModalProcessor):
811
mm_data,
812
hf_processor_mm_kwargs,
813
tokenization_kwargs,
813
- mm_hash_overrides=mm_hash_overrides)
814
+ mm_uuids=mm_uuids)
815
816
mm_items = self._to_mm_items(mm_data)
817
mm_item_counts = mm_items.get_all_counts()
vllm/model_executor/models/mllama.py
+3
-3
@@ -57,7 +57,7 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
57
from vllm.multimodal import MULTIMODAL_REGISTRY
58
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalEncDecInputs,
59
MultiModalFieldConfig,
60
- MultiModalKwargsItems)
60
+ MultiModalKwargsItems, MultiModalUUIDDict)
61
from vllm.multimodal.parse import (ImageProcessorItems, ImageSize,
62
MultiModalDataItems)
63
from vllm.multimodal.processing import (BaseProcessingInfo,
@@ -184,13 +184,13 @@ class MllamaMultiModalProcessor(EncDecMultiModalProcessor[MllamaProcessingInfo]
184
mm_data: MultiModalDataDict,
185
hf_processor_mm_kwargs: Mapping[str, object],
186
tokenization_kwargs: Optional[Mapping[str, object]] = None,
187
- mm_hash_overrides: Optional[dict[str, list[str]]] = None,
187
+ mm_uuids: Optional[MultiModalUUIDDict] = None,
188
) -> MultiModalEncDecInputs:
189
mm_inputs = super().apply(prompt,
190
mm_data,
191
hf_processor_mm_kwargs,
192
tokenization_kwargs,
193
- mm_hash_overrides=mm_hash_overrides)
193
+ mm_uuids=mm_uuids)
194
195
image_token_id = self.info.get_hf_config().image_token_index
196
# Check that the number of image tokens in the decoder prompt matches
vllm/model_executor/models/paligemma.py
+4
-3
@@ -12,7 +12,8 @@ from vllm.logger import init_logger
12
from vllm.model_executor.sampling_metadata import SamplingMetadata
13
from vllm.multimodal import MULTIMODAL_REGISTRY
14
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
15
- MultiModalInputs, MultiModalKwargsItems)
15
+ MultiModalInputs, MultiModalKwargsItems,
16
+ MultiModalUUIDDict)
17
from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
18
MultiModalDataItems)
19
from vllm.multimodal.processing import (BaseMultiModalProcessor,
@@ -203,13 +204,13 @@ class PaliGemmaMultiModalProcessor(
204
mm_data: MultiModalDataDict,
205
hf_processor_mm_kwargs: Mapping[str, object],
206
tokenization_kwargs: Optional[Mapping[str, object]] = None,
206
- mm_hash_overrides: Optional[dict[str, list[str]]] = None,
207
+ mm_uuids: Optional[MultiModalUUIDDict] = None,
208
) -> MultiModalInputs:
209
mm_inputs = super().apply(prompt,
210
mm_data,
211
hf_processor_mm_kwargs,
212
tokenization_kwargs,
212
- mm_hash_overrides=mm_hash_overrides)
213
+ mm_uuids=mm_uuids)
214
prompt_token_ids = mm_inputs["prompt_token_ids"]
215
216
tokenizer = self.info.get_tokenizer()
vllm/model_executor/models/pixtral.py
+3
-3
@@ -35,7 +35,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
35
from vllm.model_executor.sampling_metadata import SamplingMetadata
36
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargsItems
37
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
38
- NestedTensors)
38
+ MultiModalUUIDDict, NestedTensors)
39
from vllm.multimodal.parse import (ImageProcessorItems, ImageSize,
40
MultiModalDataItems)
41
from vllm.multimodal.processing import (BaseMultiModalProcessor,
@@ -316,14 +316,14 @@ class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo]
316
mm_data_items: MultiModalDataItems,
317
hf_processor_mm_kwargs: Mapping[str, object],
318
tokenization_kwargs: Mapping[str, object],
319
- mm_hash_overrides: Optional[dict[str, list[str]]] = None,
319
+ mm_uuids: Optional[MultiModalUUIDDict] = None,
320
) -> tuple[list[int], MultiModalProcessingInfo, bool]:
321
prompt_ids, mm_info, _ = super()._cached_apply_hf_processor(
322
prompt=prompt,
323
mm_data_items=mm_data_items,
324
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
325
tokenization_kwargs=tokenization_kwargs,
326
- mm_hash_overrides=mm_hash_overrides,
326
+ mm_uuids=mm_uuids,
327
)
328
329
# NOTE: The tokens are already inserted by the chat template
vllm/model_executor/models/terratorch.py
+3
-3
@@ -36,7 +36,7 @@ from vllm.multimodal.cache import MultiModalProcessorOnlyCache
36
from vllm.multimodal.inputs import (ImageItem, ModalityData,
37
MultiModalDataDict, MultiModalFieldConfig,
38
MultiModalInputs, MultiModalKwargsItems,
39
- PlaceholderRange)
39
+ MultiModalUUIDDict, PlaceholderRange)
40
from vllm.multimodal.parse import (DictEmbeddingItems, ModalityDataItems,
41
MultiModalDataItems, MultiModalDataParser)
42
from vllm.multimodal.processing import (BaseMultiModalProcessor,
@@ -164,7 +164,7 @@ class TerratorchMultiModalProcessor(BaseMultiModalProcessor):
164
mm_data: MultiModalDataDict,
165
hf_processor_mm_kwargs: Mapping[str, object],
166
tokenization_kwargs: Optional[Mapping[str, object]] = None,
167
- mm_hash_overrides: Optional[dict[str, list[str]]] = None,
167
+ mm_uuids: Optional[MultiModalUUIDDict] = None,
168
) -> MultiModalInputs:
169
if "image" in mm_data:
170
image_data = mm_data["image"]
@@ -177,7 +177,7 @@ class TerratorchMultiModalProcessor(BaseMultiModalProcessor):
177
mm_hashes = self._hash_mm_items(mm_items,
178
hf_processor_mm_kwargs,
179
tokenization_kwargs,
180
- mm_hash_overrides=mm_hash_overrides)
180
+ mm_uuids=mm_uuids)
181
mm_placeholders = {"image": [PlaceholderRange(offset=0, length=0)]}
182
183
mm_processed_data = BatchFeature(image_data)
vllm/model_executor/models/transformers.py
+5
-5
@@ -44,7 +44,8 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
44
from vllm.model_executor.sampling_metadata import SamplingMetadata
45
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargsItems
46
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
47
- MultiModalInputs, PlaceholderRange)
47
+ MultiModalInputs, MultiModalUUIDDict,
48
+ PlaceholderRange)
49
from vllm.multimodal.parse import ImageProcessorItems, MultiModalDataItems
50
from vllm.multimodal.processing import (BaseMultiModalProcessor,
51
BaseProcessingInfo)
@@ -347,7 +348,7 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
348
mm_data: MultiModalDataDict,
349
hf_processor_mm_kwargs: Mapping[str, object],
350
tokenization_kwargs: Optional[Mapping[str, object]] = None,
350
- mm_hash_overrides: Optional[dict[str, list[str]]] = None,
351
+ mm_uuids: Optional[MultiModalUUIDDict] = None,
352
) -> MultiModalInputs:
353
"""
354
Process multi-modal inputs to be used in vLLM.
@@ -415,9 +416,8 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
416
num_image_patches),
417
)
418
# Use overrides if provided; fallback to data-dependent hashing.
418
- mm_hashes = (mm_hash_overrides if mm_hash_overrides is not None else
419
- self._hash_mm_items(mm_items, hf_processor_mm_kwargs,
420
- tokenization_kwargs))
419
+ mm_hashes = (mm_uuids if mm_uuids is not None else self._hash_mm_items(
420
+ mm_items, hf_processor_mm_kwargs, tokenization_kwargs))
421
422
return MultiModalInputs(
423
type="multimodal",
vllm/model_executor/models/voxtral.py
+4
-3
@@ -31,7 +31,8 @@ from vllm.model_executor.models.whisper import WhisperEncoder
31
from vllm.model_executor.sampling_metadata import SamplingMetadata
32
from vllm.multimodal import MULTIMODAL_REGISTRY
33
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
34
- MultiModalKwargsItems, NestedTensors)
34
+ MultiModalKwargsItems, MultiModalUUIDDict,
35
+ NestedTensors)
36
from vllm.multimodal.parse import (AudioProcessorItems, MultiModalDataItems,
37
MultiModalDataParser)
38
from vllm.multimodal.processing import (BaseMultiModalProcessor,
@@ -290,14 +291,14 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo]
291
mm_data_items: MultiModalDataItems,
292
hf_processor_mm_kwargs: Mapping[str, object],
293
tokenization_kwargs: Mapping[str, object],
293
- mm_hash_overrides: Optional[dict[str, list[str]]] = None,
294
+ mm_uuids: Optional[MultiModalUUIDDict] = None,
295
) -> tuple[list[int], MultiModalProcessingInfo, bool]:
296
prompt_ids, mm_info, _ = super()._cached_apply_hf_processor(
297
prompt=prompt,
298
mm_data_items=mm_data_items,
299
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
300
tokenization_kwargs=tokenization_kwargs,
300
- mm_hash_overrides=mm_hash_overrides,
301
+ mm_uuids=mm_uuids,
302
)
303
304
# NOTE: The tokens are already inserted by the chat template
vllm/multimodal/processing.py
+22
-28
@@ -1022,13 +1022,12 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1022
mm_data: MultiModalDataDict,
1023
hf_processor_mm_kwargs: Mapping[str, object],
1024
*,
1025
- mm_hash_overrides: Optional[Union[dict[str, list[str]],
1026
- MultiModalUUIDDict]] = None,
1025
+ mm_uuids: Optional[MultiModalUUIDDict] = None,
1026
) -> MultiModalInputs:
1027
return self.apply(prompt,
1028
mm_data,
1029
hf_processor_mm_kwargs,
1031
- mm_hash_overrides=mm_hash_overrides)
1030
+ mm_uuids=mm_uuids)
1031
1032
def _get_data_parser(self) -> MultiModalDataParser:
1033
"""
@@ -1364,8 +1363,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1363
hf_processor_mm_kwargs: Mapping[str, object],
1364
tokenization_kwargs: Mapping[str, object],
1365
*,
1367
- mm_hash_overrides: Optional[Union[dict[str, list[str]],
1368
- MultiModalUUIDDict]] = None,
1366
+ mm_uuids: Optional[MultiModalUUIDDict] = None,
1367
) -> MultiModalHashes:
1368
"""Create MM hashes to be returned (only used in V1).
1369
@@ -1376,30 +1374,30 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1374
model_id = self.info.model_id
1375
1376
hashes: MultiModalHashes = {}
1379
- mm_hash_overrides = mm_hash_overrides or {}
1377
+ mm_uuids = mm_uuids or {}
1378
1379
for modality, items in mm_items.items():
1382
- if modality in mm_hash_overrides:
1383
- mm_hashes = mm_hash_overrides[modality]
1384
- if isinstance(mm_hashes, str):
1385
- mm_hashes = [mm_hashes]
1380
+ if modality in mm_uuids:
1381
+ mm_uuids_per_modality = mm_uuids[modality]
1382
+ if isinstance(mm_uuids_per_modality, str):
1383
+ mm_uuids_per_modality = [mm_uuids_per_modality]
1384
1385
# For None entries, compute a hash; otherwise, use provided ID.
1386
computed: list[str] = []
1387
for i, item in enumerate(items):
1390
- mm_hash = mm_hashes[i]
1388
+ item_uuid = mm_uuids_per_modality[i]
1389
1392
- # NOTE: Even if a mm_hash is provided, we still compute a
1390
+ # NOTE: Even if a item_uuid is provided, we still compute a
1391
# hash if `hf_processor_mm_kwargs` or `tokenization_kwargs`
1392
# are provided. This is because the processed multimodal
1393
# inputs can be different depending on the processor kwargs.
1396
- if mm_hash is None or \
1394
+ if item_uuid is None or \
1395
hf_processor_mm_kwargs or \
1396
tokenization_kwargs:
1397
1398
# NOTE: use provided hash string to hash with kwargs
1399
# if available for better performance.
1402
- item = mm_hash if mm_hash is not None else item
1400
+ item = item_uuid if item_uuid is not None else item
1401
computed.append(
1402
MultiModalHasher.hash_kwargs(
1403
model_id=model_id,
@@ -1407,7 +1405,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1405
**hf_processor_mm_kwargs,
1406
**tokenization_kwargs))
1407
else:
1410
- computed.append(mm_hash)
1408
+ computed.append(item_uuid)
1409
hashes[modality] = computed
1410
else:
1411
hashes[modality] = [
@@ -1514,8 +1512,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1512
hf_processor_mm_kwargs: Mapping[str, object],
1513
tokenization_kwargs: Mapping[str, object],
1514
*,
1517
- mm_hash_overrides: Optional[Union[dict[str, list[str]],
1518
- MultiModalUUIDDict]] = None,
1515
+ mm_uuids: Optional[MultiModalUUIDDict] = None,
1516
) -> tuple[list[int], MultiModalProcessingInfo, bool]:
1517
(
1518
prompt_ids,
@@ -1539,7 +1536,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1536
mm_hashes = self._hash_mm_items(mm_data_items,
1537
hf_processor_mm_kwargs,
1538
tokenization_kwargs,
1542
- mm_hash_overrides=mm_hash_overrides)
1539
+ mm_uuids=mm_uuids)
1540
1541
mm_prompt_updates = self._get_mm_prompt_updates(
1542
mm_data_items,
@@ -1562,8 +1559,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1559
hf_processor_mm_kwargs: Mapping[str, object],
1560
tokenization_kwargs: Mapping[str, object],
1561
*,
1565
- mm_hash_overrides: Optional[Union[dict[str, list[str]],
1566
- MultiModalUUIDDict]] = None,
1562
+ mm_uuids: Optional[MultiModalUUIDDict] = None,
1563
) -> tuple[list[int], MultiModalProcessingInfo, bool]:
1564
"""
1565
Apply the HF processor on the full prompt text,
@@ -1578,13 +1574,13 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1574
mm_data_items=mm_data_items,
1575
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1576
tokenization_kwargs=tokenization_kwargs,
1581
- mm_hash_overrides=mm_hash_overrides,
1577
+ mm_uuids=mm_uuids,
1578
)
1579
1580
mm_hashes = self._hash_mm_items(mm_data_items,
1581
hf_processor_mm_kwargs,
1582
tokenization_kwargs,
1587
- mm_hash_overrides=mm_hash_overrides)
1583
+ mm_uuids=mm_uuids)
1584
1585
mm_missing_data_items = self._get_cache_missing_items(
1586
cache=cache,
@@ -1785,8 +1781,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1781
hf_processor_mm_kwargs: Mapping[str, object],
1782
tokenization_kwargs: Optional[Mapping[str, object]] = None,
1783
*,
1788
- mm_hash_overrides: Optional[Union[dict[str, list[str]],
1789
- MultiModalUUIDDict]] = None,
1784
+ mm_uuids: Optional[MultiModalUUIDDict] = None,
1785
) -> MultiModalInputs:
1786
"""
1787
Process multi-modal inputs to be used in vLLM.
@@ -1815,7 +1810,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1810
mm_items,
1811
hf_processor_mm_kwargs,
1812
tokenization_kwargs=tokenization_kwargs,
1818
- mm_hash_overrides=mm_hash_overrides,
1813
+ mm_uuids=mm_uuids,
1814
)
1815
1816
# NOTE: tokenization_kwargs are not required to init processor
@@ -1901,8 +1896,7 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
1896
hf_processor_mm_kwargs: Mapping[str, object],
1897
tokenization_kwargs: Optional[Mapping[str, object]] = None,
1898
*,
1904
- mm_hash_overrides: Optional[Union[dict[str, list[str]],
1905
- MultiModalUUIDDict]] = None,
1899
+ mm_uuids: Optional[MultiModalUUIDDict] = None,
1900
) -> MultiModalEncDecInputs:
1901
"""
1902
Process multi-modal inputs to be used in vLLM.
@@ -1917,7 +1911,7 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
1911
mm_data,
1912
hf_processor_mm_kwargs,
1913
tokenization_kwargs,
1920
- mm_hash_overrides=mm_hash_overrides,
1914
+ mm_uuids=mm_uuids,
1915
)
1916
1917
return self._get_enc_dec_inputs(
vllm/v1/engine/processor.py
+10
-11
@@ -12,7 +12,7 @@ from vllm.inputs.preprocess import InputPreprocessor
12
from vllm.lora.request import LoRARequest
13
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
14
from vllm.multimodal.cache import processor_cache_from_config
15
-from vllm.multimodal.inputs import MultiModalFeatureSpec
15
+from vllm.multimodal.inputs import MultiModalFeatureSpec, MultiModalUUIDDict
16
from vllm.multimodal.processing import EncDecMultiModalProcessor
17
from vllm.multimodal.utils import argsort_mm_positions
18
from vllm.pooling_params import PoolingParams
@@ -276,11 +276,11 @@ class Processor:
276
# Remember that this backend was set automatically
277
params.guided_decoding.backend_was_auto = True
278
279
- def _maybe_build_mm_hash_overrides(
279
+ def _maybe_build_mm_uuids(
280
self,
281
request_id: str,
282
prompt: PromptType,
283
- ) -> Optional[dict[str, list[str]]]:
283
+ ) -> Optional[MultiModalUUIDDict]:
284
"""Build per-item multimodal hash overrides when enabled. In this case,
285
multimodal data items are identified by their request id, modality and
286
index rather than their content.
@@ -303,13 +303,13 @@ class Processor:
303
if not mm_data:
304
return None
305
306
- overrides: dict[str, list[str]] = {}
306
+ mm_uuids: MultiModalUUIDDict = {}
307
for modality, data in mm_data.items():
308
n = len(data) if isinstance(data, list) else 1
309
- overrides[modality] = [
309
+ mm_uuids[modality] = [
310
f"{request_id}-{modality}-{i}" for i in range(n)
311
]
312
- return overrides
312
+ return mm_uuids
313
314
def process_inputs(
315
self,
@@ -351,16 +351,15 @@ class Processor:
351
if (self.model_config.multimodal_config and
352
self.model_config.multimodal_config.mm_processor_cache_gb == 0
353
and not self.cache_config.enable_prefix_caching):
354
- mm_hash_overrides = self._maybe_build_mm_hash_overrides(
355
- request_id, prompt)
354
+ mm_uuids = self._maybe_build_mm_uuids(request_id, prompt)
355
else:
356
# Otherwise, use user-provided uuids as multimodal hash overrides
357
# if provided.
358
self._validate_multi_modal_uuids(prompt)
359
if isinstance(prompt, dict):
361
- mm_hash_overrides = prompt.get("multi_modal_uuids")
360
+ mm_uuids = prompt.get("multi_modal_uuids")
361
else:
363
- mm_hash_overrides = None
362
+ mm_uuids = None
363
364
# Process inputs, which includes:
365
# 1. Tokenize text prompt, with LoRA request if one exists.
@@ -370,7 +369,7 @@ class Processor:
369
prompt,
370
tokenization_kwargs=tokenization_kwargs,
371
lora_request=lora_request,
373
- mm_hash_overrides=mm_hash_overrides,
372
+ mm_uuids=mm_uuids,
373
)
374
from vllm.platforms import current_platform
375
current_platform.validate_request(