A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Multimodal] Generate mm_hash based on request metadata when caching is turned off (#23690)
Signed-off-by: Roger Wang <hey@rogerw.io>
Roger Wang committed
Aug 27, 2025 at 13:24 UTC
8bf6266a17933b130f94f6d53f32ac029ed8ba1b
12 files changed
+179
-24
vllm/inputs/preprocess.py
+63
-8
@@ -257,6 +257,8 @@ class InputPreprocessor:
257
mm_processor_kwargs: Optional[Mapping[str, object]],
258
tokenization_kwargs: Optional[dict[str, Any]] = None,
259
lora_request: Optional[LoRARequest] = None,
260
+ *,
261
+ mm_hash_overrides: Optional[dict[str, list[str]]] = None,
262
) -> MultiModalInputs:
263
"""
264
Apply the model's multi-modal processor to a multi-modal prompt,
@@ -273,10 +275,13 @@ class InputPreprocessor:
275
if mm_processor_kwargs is None:
276
mm_processor_kwargs = {}
277
276
- return mm_processor.apply(prompt,
277
- mm_data,
278
- hf_processor_mm_kwargs=mm_processor_kwargs,
279
- tokenization_kwargs=tokenization_kwargs)
278
+ return mm_processor.apply(
279
+ prompt,
280
+ mm_data,
281
+ hf_processor_mm_kwargs=mm_processor_kwargs,
282
+ tokenization_kwargs=tokenization_kwargs,
283
+ mm_hash_overrides=mm_hash_overrides,
284
+ )
285
286
async def _process_multimodal_async(
287
self,
@@ -285,6 +290,8 @@ class InputPreprocessor:
290
mm_processor_kwargs: Optional[Mapping[str, object]],
291
tokenization_kwargs: Optional[dict[str, Any]] = None,
292
lora_request: Optional[LoRARequest] = None,
293
+ *,
294
+ mm_hash_overrides: Optional[dict[str, list[str]]] = None,
295
) -> MultiModalInputs:
296
"""
297
Async version of
@@ -301,10 +308,13 @@ class InputPreprocessor:
308
if mm_processor_kwargs is None:
309
mm_processor_kwargs = {}
310
304
- return mm_processor.apply(prompt,
305
- mm_data,
306
- hf_processor_mm_kwargs=mm_processor_kwargs,
307
- tokenization_kwargs=tokenization_kwargs)
311
+ return mm_processor.apply(
312
+ prompt,
313
+ mm_data,
314
+ hf_processor_mm_kwargs=mm_processor_kwargs,
315
+ tokenization_kwargs=tokenization_kwargs,
316
+ mm_hash_overrides=mm_hash_overrides,
317
+ )
318
319
def _process_embeds(
320
self,
@@ -341,6 +351,8 @@ class InputPreprocessor:
351
parsed_content: TokensPrompt,
352
tokenization_kwargs: Optional[dict[str, Any]] = None,
353
lora_request: Optional[LoRARequest] = None,
354
+ *,
355
+ mm_hash_overrides: Optional[dict[str, list[str]]] = None,
356
) -> Union[TokenInputs, MultiModalInputs]:
357
prompt_token_ids = parsed_content["prompt_token_ids"]
358
token_type_ids = parsed_content.get("token_type_ids")
@@ -353,6 +365,7 @@ class InputPreprocessor:
365
parsed_content.get("mm_processor_kwargs"),
366
tokenization_kwargs=tokenization_kwargs,
367
lora_request=lora_request,
368
+ mm_hash_overrides=mm_hash_overrides,
369
)
370
else:
371
inputs = token_inputs(
@@ -370,6 +383,8 @@ class InputPreprocessor:
383
parsed_content: TokensPrompt,
384
tokenization_kwargs: Optional[dict[str, Any]] = None,
385
lora_request: Optional[LoRARequest] = None,
386
+ *,
387
+ mm_hash_overrides: Optional[dict[str, list[str]]] = None,
388
) -> Union[TokenInputs, MultiModalInputs]:
389
prompt_token_ids = parsed_content["prompt_token_ids"]
390
token_type_ids = parsed_content.get("token_type_ids")
@@ -382,6 +397,7 @@ class InputPreprocessor:
397
parsed_content.get("mm_processor_kwargs"),
398
tokenization_kwargs=tokenization_kwargs,
399
lora_request=lora_request,
400
+ mm_hash_overrides=mm_hash_overrides,
401
)
402
else:
403
inputs = token_inputs(
@@ -399,6 +415,8 @@ class InputPreprocessor:
415
parsed_content: TextPrompt,
416
tokenization_kwargs: Optional[dict[str, Any]] = None,
417
lora_request: Optional[LoRARequest] = None,
418
+ *,
419
+ mm_hash_overrides: Optional[dict[str, list[str]]] = None,
420
) -> Union[TokenInputs, MultiModalInputs]:
421
prompt_text = parsed_content["prompt"]
422
@@ -410,6 +428,7 @@ class InputPreprocessor:
428
parsed_content.get("mm_processor_kwargs"),
429
tokenization_kwargs=tokenization_kwargs,
430
lora_request=lora_request,
431
+ mm_hash_overrides=mm_hash_overrides,
432
)
433
else:
434
prompt_token_ids = self._tokenize_prompt(
@@ -432,6 +451,8 @@ class InputPreprocessor:
451
parsed_content: TextPrompt,
452
tokenization_kwargs: Optional[dict[str, Any]] = None,
453
lora_request: Optional[LoRARequest] = None,
454
+ *,
455
+ mm_hash_overrides: Optional[dict[str, list[str]]] = None,
456
) -> Union[TokenInputs, MultiModalInputs]:
457
prompt_text = parsed_content["prompt"]
458
@@ -443,6 +464,7 @@ class InputPreprocessor:
464
parsed_content.get("mm_processor_kwargs"),
465
tokenization_kwargs=tokenization_kwargs,
466
lora_request=lora_request,
467
+ mm_hash_overrides=mm_hash_overrides,
468
)
469
else:
470
prompt_token_ids = await self._tokenize_prompt_async(
@@ -465,6 +487,8 @@ class InputPreprocessor:
487
prompt: SingletonPrompt,
488
tokenization_kwargs: Optional[dict[str, Any]] = None,
489
lora_request: Optional[LoRARequest] = None,
490
+ *,
491
+ mm_hash_overrides: Optional[dict[str, list[str]]] = None,
492
) -> SingletonInputs:
493
"""
494
Extract the singleton inputs from a prompt.
@@ -486,18 +510,21 @@ class InputPreprocessor:
510
return self._process_tokens(
511
parsed["content"],
512
lora_request=lora_request,
513
+ mm_hash_overrides=mm_hash_overrides,
514
)
515
if parsed["type"] == "text":
516
return self._process_text(
517
parsed["content"],
518
tokenization_kwargs=tokenization_kwargs,
519
lora_request=lora_request,
520
+ mm_hash_overrides=mm_hash_overrides,
521
)
522
if parsed["type"] == "str":
523
return self._process_text(
524
TextPrompt(prompt=parsed["content"]),
525
tokenization_kwargs=tokenization_kwargs,
526
lora_request=lora_request,
527
+ mm_hash_overrides=mm_hash_overrides,
528
)
529
530
assert_never(parsed)
@@ -507,6 +534,8 @@ class InputPreprocessor:
534
prompt: SingletonPrompt,
535
tokenization_kwargs: Optional[dict[str, Any]] = None,
536
lora_request: Optional[LoRARequest] = None,
537
+ *,
538
+ mm_hash_overrides: Optional[dict[str, list[str]]] = None,
539
) -> SingletonInputs:
540
"""
541
Async version of
@@ -520,18 +549,21 @@ class InputPreprocessor:
549
return await self._process_tokens_async(
550
parsed["content"],
551
lora_request=lora_request,
552
+ mm_hash_overrides=mm_hash_overrides,
553
)
554
if parsed["type"] == "text":
555
return await self._process_text_async(
556
parsed["content"],
557
tokenization_kwargs=tokenization_kwargs,
558
lora_request=lora_request,
559
+ mm_hash_overrides=mm_hash_overrides,
560
)
561
if parsed["type"] == "str":
562
return await self._process_text_async(
563
TextPrompt(prompt=parsed["content"]),
564
tokenization_kwargs=tokenization_kwargs,
565
lora_request=lora_request,
566
+ mm_hash_overrides=mm_hash_overrides,
567
)
568
569
assert_never(parsed)
@@ -641,6 +673,8 @@ class InputPreprocessor:
673
self,
674
prompt: PromptType,
675
tokenization_kwargs: Optional[dict[str, Any]] = None,
676
+ *,
677
+ mm_hash_overrides: Optional[dict[str, list[str]]] = None,
678
) -> EncoderDecoderInputs:
679
"""
680
For encoder/decoder models only:
@@ -682,6 +716,7 @@ class InputPreprocessor:
716
encoder_inputs = self._prompt_to_llm_inputs(
717
prompt["encoder_prompt"],
718
tokenization_kwargs=tokenization_kwargs,
719
+ mm_hash_overrides=mm_hash_overrides,
720
)
721
if (decoder_input := prompt["decoder_prompt"]) is None:
722
decoder_inputs = None
@@ -697,6 +732,7 @@ class InputPreprocessor:
732
inputs = self._prompt_to_llm_inputs(
733
prompt,
734
tokenization_kwargs=tokenization_kwargs,
735
+ mm_hash_overrides=mm_hash_overrides,
736
)
737
if self.model_config.is_multimodal_model:
738
# Encoder-Decoder Multimodal model
@@ -712,6 +748,8 @@ class InputPreprocessor:
748
self,
749
prompt: PromptType,
750
tokenization_kwargs: Optional[dict[str, Any]] = None,
751
+ *,
752
+ mm_hash_overrides: Optional[dict[str, list[str]]] = None,
753
) -> EncoderDecoderInputs:
754
"""
755
Async version of
@@ -724,6 +762,7 @@ class InputPreprocessor:
762
encoder_task = self._prompt_to_llm_inputs_async(
763
prompt["encoder_prompt"],
764
tokenization_kwargs=tokenization_kwargs,
765
+ mm_hash_overrides=mm_hash_overrides,
766
)
767
768
if (decoder_input := prompt["decoder_prompt"]) is None:
@@ -733,6 +772,7 @@ class InputPreprocessor:
772
decoder_task = self._prompt_to_llm_inputs_async(
773
decoder_input,
774
tokenization_kwargs=tokenization_kwargs,
775
+ mm_hash_overrides=mm_hash_overrides,
776
)
777
778
encoder_inputs, decoder_inputs = await asyncio.gather(
@@ -748,6 +788,7 @@ class InputPreprocessor:
788
inputs = await self._prompt_to_llm_inputs_async(
789
prompt,
790
tokenization_kwargs=tokenization_kwargs,
791
+ mm_hash_overrides=mm_hash_overrides,
792
)
793
if self.model_config.is_multimodal_model:
794
# Encoder-Decoder Multimodal model
@@ -774,6 +815,8 @@ class InputPreprocessor:
815
prompt: SingletonPrompt,
816
tokenization_kwargs: Optional[dict[str, Any]] = None,
817
lora_request: Optional[LoRARequest] = None,
818
+ *,
819
+ mm_hash_overrides: Optional[dict[str, list[str]]] = None,
820
) -> DecoderOnlyInputs:
821
"""
822
For decoder-only models:
@@ -794,6 +837,7 @@ class InputPreprocessor:
837
prompt,
838
tokenization_kwargs=tokenization_kwargs,
839
lora_request=lora_request,
840
+ mm_hash_overrides=mm_hash_overrides,
841
)
842
843
return self._build_decoder_only_llm_inputs(prompt_comps)
@@ -803,6 +847,8 @@ class InputPreprocessor:
847
prompt: SingletonPrompt,
848
tokenization_kwargs: Optional[dict[str, Any]] = None,
849
lora_request: Optional[LoRARequest] = None,
850
+ *,
851
+ mm_hash_overrides: Optional[dict[str, list[str]]] = None,
852
) -> DecoderOnlyInputs:
853
"""
854
Async version of
@@ -812,6 +858,7 @@ class InputPreprocessor:
858
prompt,
859
tokenization_kwargs=tokenization_kwargs,
860
lora_request=lora_request,
861
+ mm_hash_overrides=mm_hash_overrides,
862
)
863
864
return self._build_decoder_only_llm_inputs(prompt_comps)
@@ -821,6 +868,8 @@ class InputPreprocessor:
868
prompt: PromptType,
869
tokenization_kwargs: Optional[dict[str, Any]] = None,
870
lora_request: Optional[LoRARequest] = None,
871
+ *,
872
+ mm_hash_overrides: Optional[dict[str, list[str]]] = None,
873
) -> ProcessorInputs:
874
"""Preprocess the input prompt."""
875
if self.model_config.is_encoder_decoder:
@@ -829,6 +878,7 @@ class InputPreprocessor:
878
return self._process_encoder_decoder_prompt(
879
prompt,
880
tokenization_kwargs,
881
+ mm_hash_overrides=mm_hash_overrides,
882
)
883
884
if is_explicit_encoder_decoder_prompt(prompt):
@@ -840,6 +890,7 @@ class InputPreprocessor:
890
prompt,
891
tokenization_kwargs=tokenization_kwargs,
892
lora_request=lora_request,
893
+ mm_hash_overrides=mm_hash_overrides,
894
)
895
896
async def preprocess_async(
@@ -847,6 +898,8 @@ class InputPreprocessor:
898
prompt: PromptType,
899
tokenization_kwargs: Optional[dict[str, Any]] = None,
900
lora_request: Optional[LoRARequest] = None,
901
+ *,
902
+ mm_hash_overrides: Optional[dict[str, list[str]]] = None,
903
) -> ProcessorInputs:
904
"""
905
Async version of
@@ -858,6 +911,7 @@ class InputPreprocessor:
911
return await self._process_encoder_decoder_prompt_async(
912
prompt,
913
tokenization_kwargs,
914
+ mm_hash_overrides=mm_hash_overrides,
915
)
916
917
if is_explicit_encoder_decoder_prompt(prompt):
@@ -869,6 +923,7 @@ class InputPreprocessor:
923
prompt,
924
tokenization_kwargs=tokenization_kwargs,
925
lora_request=lora_request,
926
+ mm_hash_overrides=mm_hash_overrides,
927
)
928
929
def clear_cache(self) -> None:
vllm/model_executor/models/deepseek_vl2.py
+3
@@ -290,6 +290,7 @@ class DeepseekVL2MultiModalProcessor(
290
mm_data_items: MultiModalDataItems,
291
hf_processor_mm_kwargs: Mapping[str, object],
292
tokenization_kwargs: Mapping[str, object],
293
+ mm_hash_overrides: Optional[dict[str, list[str]]] = None,
294
) -> tuple[list[int], MultiModalProcessingInfo, bool]:
295
# The processor logic is different for len(images) <= 2 vs > 2
296
# Since the processing cache assumes that the processor output is
@@ -301,6 +302,7 @@ class DeepseekVL2MultiModalProcessor(
302
mm_data_items=mm_data_items,
303
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
304
tokenization_kwargs=tokenization_kwargs,
305
+ mm_hash_overrides=mm_hash_overrides,
306
)
307
308
return super()._cached_apply_hf_processor(
@@ -308,6 +310,7 @@ class DeepseekVL2MultiModalProcessor(
310
mm_data_items=mm_data_items,
311
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
312
tokenization_kwargs=tokenization_kwargs,
313
+ mm_hash_overrides=mm_hash_overrides,
314
)
315
316
vllm/model_executor/models/h2ovl.py
+3
@@ -479,6 +479,7 @@ class H2OVLMultiModalProcessor(
479
mm_data_items: MultiModalDataItems,
480
hf_processor_mm_kwargs: Mapping[str, object],
481
tokenization_kwargs: Mapping[str, object],
482
+ mm_hash_overrides: Optional[dict[str, list[str]]] = None,
483
) -> tuple[list[int], MultiModalProcessingInfo, bool]:
484
# The processor logic is different for len(images) <= 1 vs > 1
485
# Since the processing cache assumes that the processor output is
@@ -490,6 +491,7 @@ class H2OVLMultiModalProcessor(
491
mm_data_items=mm_data_items,
492
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
493
tokenization_kwargs=tokenization_kwargs,
494
+ mm_hash_overrides=mm_hash_overrides,
495
)
496
497
return super()._cached_apply_hf_processor(
@@ -497,6 +499,7 @@ class H2OVLMultiModalProcessor(
499
mm_data_items=mm_data_items,
500
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
501
tokenization_kwargs=tokenization_kwargs,
502
+ mm_hash_overrides=mm_hash_overrides,
503
)
504
505
vllm/model_executor/models/llava.py
+6
-2
@@ -795,6 +795,7 @@ class MantisMultiModalProcessor(LlavaMultiModalProcessor):
795
mm_data: MultiModalDataDict,
796
hf_processor_mm_kwargs: Mapping[str, object],
797
tokenization_kwargs: Optional[Mapping[str, object]] = None,
798
+ mm_hash_overrides: Optional[dict[str, list[str]]] = None,
799
) -> MultiModalInputs:
800
hf_config = self.info.get_hf_config()
801
image_token_id = hf_config.image_token_index
@@ -805,8 +806,11 @@ class MantisMultiModalProcessor(LlavaMultiModalProcessor):
806
image_height=-1,
807
)
808
808
- result = super().apply(prompt, mm_data, hf_processor_mm_kwargs,
809
- tokenization_kwargs)
809
+ result = super().apply(prompt,
810
+ mm_data,
811
+ hf_processor_mm_kwargs,
812
+ tokenization_kwargs,
813
+ mm_hash_overrides=mm_hash_overrides)
814
815
mm_items = self._to_mm_items(mm_data)
816
mm_item_counts = mm_items.get_all_counts()
vllm/model_executor/models/mllama.py
+6
-2
@@ -184,9 +184,13 @@ class MllamaMultiModalProcessor(EncDecMultiModalProcessor[MllamaProcessingInfo]
184
mm_data: MultiModalDataDict,
185
hf_processor_mm_kwargs: Mapping[str, object],
186
tokenization_kwargs: Optional[Mapping[str, object]] = None,
187
+ mm_hash_overrides: Optional[dict[str, list[str]]] = None,
188
) -> MultiModalEncDecInputs:
188
- mm_inputs = super().apply(prompt, mm_data, hf_processor_mm_kwargs,
189
- tokenization_kwargs)
189
+ mm_inputs = super().apply(prompt,
190
+ mm_data,
191
+ hf_processor_mm_kwargs,
192
+ tokenization_kwargs,
193
+ mm_hash_overrides=mm_hash_overrides)
194
195
image_token_id = self.info.get_hf_config().image_token_index
196
# Check that the number of image tokens in the decoder prompt matches
vllm/model_executor/models/paligemma.py
+6
-2
@@ -203,9 +203,13 @@ class PaliGemmaMultiModalProcessor(
203
mm_data: MultiModalDataDict,
204
hf_processor_mm_kwargs: Mapping[str, object],
205
tokenization_kwargs: Optional[Mapping[str, object]] = None,
206
+ mm_hash_overrides: Optional[dict[str, list[str]]] = None,
207
) -> MultiModalInputs:
207
- mm_inputs = super().apply(prompt, mm_data, hf_processor_mm_kwargs,
208
- tokenization_kwargs)
208
+ mm_inputs = super().apply(prompt,
209
+ mm_data,
210
+ hf_processor_mm_kwargs,
211
+ tokenization_kwargs,
212
+ mm_hash_overrides=mm_hash_overrides)
213
prompt_token_ids = mm_inputs["prompt_token_ids"]
214
215
tokenizer = self.info.get_tokenizer()
vllm/model_executor/models/pixtral.py
+2
@@ -314,12 +314,14 @@ class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo]
314
mm_data_items: MultiModalDataItems,
315
hf_processor_mm_kwargs: Mapping[str, object],
316
tokenization_kwargs: Mapping[str, object],
317
+ mm_hash_overrides: Optional[dict[str, list[str]]] = None,
318
) -> tuple[list[int], MultiModalProcessingInfo, bool]:
319
prompt_ids, mm_info, _ = super()._cached_apply_hf_processor(
320
prompt=prompt,
321
mm_data_items=mm_data_items,
322
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
323
tokenization_kwargs=tokenization_kwargs,
324
+ mm_hash_overrides=mm_hash_overrides,
325
)
326
327
# NOTE: The tokens are already inserted by the chat template
vllm/model_executor/models/prithvi_geospatial_mae.py
+5
-2
@@ -138,6 +138,7 @@ class PrithviGeoSpatialMAEMultiModalProcessor(BaseMultiModalProcessor):
138
mm_data: MultiModalDataDict,
139
hf_processor_mm_kwargs: Mapping[str, object],
140
tokenization_kwargs: Optional[Mapping[str, object]] = None,
141
+ mm_hash_overrides: Optional[dict[str, list[str]]] = None,
142
) -> MultiModalInputs:
143
if "image" in mm_data:
144
image_data = mm_data["image"]
@@ -146,8 +147,10 @@ class PrithviGeoSpatialMAEMultiModalProcessor(BaseMultiModalProcessor):
147
mm_data = {"image": mm_data}
148
149
mm_items = self._to_mm_items(mm_data)
149
- mm_hashes = self._hash_mm_items(mm_items, hf_processor_mm_kwargs,
150
- tokenization_kwargs or {})
150
+ tokenization_kwargs = tokenization_kwargs or {}
151
+ mm_hashes = (mm_hash_overrides if mm_hash_overrides is not None else
152
+ self._hash_mm_items(mm_items, hf_processor_mm_kwargs,
153
+ tokenization_kwargs))
154
mm_placeholders = {"image": [PlaceholderRange(offset=0, length=0)]}
155
156
mm_processed_data = BatchFeature(image_data)
vllm/model_executor/models/transformers.py
+5
-2
@@ -327,6 +327,7 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
327
mm_data: MultiModalDataDict,
328
hf_processor_mm_kwargs: Mapping[str, object],
329
tokenization_kwargs: Optional[Mapping[str, object]] = None,
330
+ mm_hash_overrides: Optional[dict[str, list[str]]] = None,
331
) -> MultiModalInputs:
332
"""
333
Process multi-modal inputs to be used in vLLM.
@@ -393,9 +394,11 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
394
self._get_mm_fields_config(processed_data, hf_processor_mm_kwargs,
395
num_image_patches),
396
)
397
+ # Use overrides if provided; fallback to data-dependent hashing.
398
+ mm_hashes = (mm_hash_overrides if mm_hash_overrides is not None else
399
+ self._hash_mm_items(mm_items, hf_processor_mm_kwargs,
400
+ tokenization_kwargs))
401
397
- mm_hashes = self._hash_mm_items(mm_items, hf_processor_mm_kwargs,
398
- tokenization_kwargs)
402
return MultiModalInputs(
403
type="multimodal",
404
prompt=prompt,
vllm/model_executor/models/voxtral.py
+2
@@ -288,12 +288,14 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo]
288
mm_data_items: MultiModalDataItems,
289
hf_processor_mm_kwargs: Mapping[str, object],
290
tokenization_kwargs: Mapping[str, object],
291
+ mm_hash_overrides: Optional[dict[str, list[str]]] = None,
292
) -> tuple[list[int], MultiModalProcessingInfo, bool]:
293
prompt_ids, mm_info, _ = super()._cached_apply_hf_processor(
294
prompt=prompt,
295
mm_data_items=mm_data_items,
296
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
297
tokenization_kwargs=tokenization_kwargs,
298
+ mm_hash_overrides=mm_hash_overrides,
299
)
300
301
# NOTE: The tokens are already inserted by the chat template
vllm/multimodal/processing.py
+30
-6
@@ -1020,8 +1020,13 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1020
prompt: str,
1021
mm_data: MultiModalDataDict,
1022
hf_processor_mm_kwargs: Mapping[str, object],
1023
+ *,
1024
+ mm_hash_overrides: Optional[MultiModalHashes] = None,
1025
) -> MultiModalInputs:
1024
- return self.apply(prompt, mm_data, hf_processor_mm_kwargs)
1026
+ return self.apply(prompt,
1027
+ mm_data,
1028
+ hf_processor_mm_kwargs,
1029
+ mm_hash_overrides=mm_hash_overrides)
1030
1031
def _get_data_parser(self) -> MultiModalDataParser:
1032
"""
@@ -1357,7 +1362,11 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1362
hf_processor_mm_kwargs: Mapping[str, object],
1363
tokenization_kwargs: Mapping[str, object],
1364
) -> MultiModalHashes:
1360
- """Create MM hashes to be returned (only used in V1)."""
1365
+ """Create MM hashes to be returned (only used in V1).
1366
+
1367
+ Note: When overrides are provided via callers of `apply`,
1368
+ `_hash_mm_items` will be bypassed and the overrides will be used.
1369
+ """
1370
model_id = self.info.model_id
1371
1372
return {
@@ -1464,6 +1473,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1473
mm_data_items: MultiModalDataItems,
1474
hf_processor_mm_kwargs: Mapping[str, object],
1475
tokenization_kwargs: Mapping[str, object],
1476
+ *,
1477
+ mm_hash_overrides: Optional[MultiModalHashes] = None,
1478
) -> tuple[list[int], MultiModalProcessingInfo, bool]:
1479
(
1480
prompt_ids,
@@ -1483,8 +1494,10 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1494
hf_processor_mm_kwargs),
1495
)
1496
1486
- mm_hashes = self._hash_mm_items(mm_data_items, hf_processor_mm_kwargs,
1487
- tokenization_kwargs)
1497
+ # Use overrides if provided; fallback to data-dependent hashing.
1498
+ mm_hashes = (mm_hash_overrides if mm_hash_overrides is not None else
1499
+ self._hash_mm_items(mm_data_items, hf_processor_mm_kwargs,
1500
+ tokenization_kwargs))
1501
1502
mm_prompt_updates = self._get_mm_prompt_updates(
1503
mm_data_items,
@@ -1506,6 +1519,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1519
mm_data_items: MultiModalDataItems,
1520
hf_processor_mm_kwargs: Mapping[str, object],
1521
tokenization_kwargs: Mapping[str, object],
1522
+ *,
1523
+ mm_hash_overrides: Optional[MultiModalHashes] = None,
1524
) -> tuple[list[int], MultiModalProcessingInfo, bool]:
1525
"""
1526
Apply the HF processor on the full prompt text,
@@ -1520,10 +1535,13 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1535
mm_data_items=mm_data_items,
1536
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1537
tokenization_kwargs=tokenization_kwargs,
1538
+ mm_hash_overrides=mm_hash_overrides,
1539
)
1540
1525
- mm_hashes = self._hash_mm_items(mm_data_items, hf_processor_mm_kwargs,
1526
- tokenization_kwargs)
1541
+ # Use overrides if provided; fallback to data-dependent hashing.
1542
+ mm_hashes = (mm_hash_overrides if mm_hash_overrides is not None else
1543
+ self._hash_mm_items(mm_data_items, hf_processor_mm_kwargs,
1544
+ tokenization_kwargs))
1545
1546
mm_missing_data_items = self._get_cache_missing_items(
1547
cache=cache,
@@ -1723,6 +1741,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1741
mm_data: MultiModalDataDict,
1742
hf_processor_mm_kwargs: Mapping[str, object],
1743
tokenization_kwargs: Optional[Mapping[str, object]] = None,
1744
+ *,
1745
+ mm_hash_overrides: Optional[dict[str, list[str]]] = None,
1746
) -> MultiModalInputs:
1747
"""
1748
Process multi-modal inputs to be used in vLLM.
@@ -1751,6 +1771,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1771
mm_items,
1772
hf_processor_mm_kwargs,
1773
tokenization_kwargs=tokenization_kwargs,
1774
+ mm_hash_overrides=mm_hash_overrides,
1775
)
1776
1777
# NOTE: tokenization_kwargs are not required to init processor
@@ -1835,6 +1856,8 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
1856
mm_data: MultiModalDataDict,
1857
hf_processor_mm_kwargs: Mapping[str, object],
1858
tokenization_kwargs: Optional[Mapping[str, object]] = None,
1859
+ *,
1860
+ mm_hash_overrides: Optional[MultiModalHashes] = None,
1861
) -> MultiModalEncDecInputs:
1862
"""
1863
Process multi-modal inputs to be used in vLLM.
@@ -1849,6 +1872,7 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
1872
mm_data,
1873
hf_processor_mm_kwargs,
1874
tokenization_kwargs,
1875
+ mm_hash_overrides=mm_hash_overrides,
1876
)
1877
1878
return self._get_enc_dec_inputs(
vllm/v1/engine/processor.py
+48
@@ -225,6 +225,41 @@ class Processor:
225
# Remember that this backend was set automatically
226
params.guided_decoding.backend_was_auto = True
227
228
+ def _maybe_build_mm_hash_overrides(
229
+ self,
230
+ request_id: str,
231
+ prompt: PromptType,
232
+ ) -> Optional[dict[str, list[str]]]:
233
+ """Build per-item multimodal hash overrides when enabled. In this case,
234
+ multimodal data items are identified by their request id, modality and
235
+ index rather than their content.
236
+
237
+ Returns a dictionary of modality -> list[str] of overrides, or None if
238
+ disabled or no multimodal data is present.
239
+ """
240
+
241
+ def _extract_mm_data(p: PromptType):
242
+ if isinstance(p, dict) and "encoder_prompt" in p:
243
+ enc = p.get("encoder_prompt")
244
+ if isinstance(enc, dict):
245
+ return enc.get("multi_modal_data")
246
+ return None
247
+ if isinstance(p, dict):
248
+ return p.get("multi_modal_data")
249
+ return None
250
+
251
+ mm_data = _extract_mm_data(prompt)
252
+ if not mm_data:
253
+ return None
254
+
255
+ overrides: dict[str, list[str]] = {}
256
+ for modality, data in mm_data.items():
257
+ n = len(data) if isinstance(data, list) else 1
258
+ overrides[modality] = [
259
+ f"{request_id}-{modality}-{i}" for i in range(n)
260
+ ]
261
+ return overrides
262
+
263
def process_inputs(
264
self,
265
request_id: str,
@@ -254,6 +289,18 @@ class Processor:
289
if arrival_time is None:
290
arrival_time = time.time()
291
292
+ # Optionally generate multimodal hash overrides based on request id.
293
+ # NOTE: when users explicitly turn off BOTH prefix caching and input
294
+ # processing caching, no multimodal features or embeddings will be
295
+ # reused across requests, therefore hashing is no longer necessary.
296
+ if (self.model_config.multimodal_config and
297
+ self.model_config.multimodal_config.mm_processor_cache_gb == 0
298
+ and not self.cache_config.enable_prefix_caching):
299
+ mm_hash_overrides = self._maybe_build_mm_hash_overrides(
300
+ request_id, prompt)
301
+ else:
302
+ mm_hash_overrides = None
303
+
304
# Process inputs, which includes:
305
# 1. Tokenize text prompt, with LoRA request if one exists.
306
# 2. For multimodal models with a merged preprocessor, preprocess
@@ -262,6 +309,7 @@ class Processor:
309
prompt,
310
tokenization_kwargs=tokenization_kwargs,
311
lora_request=lora_request,
312
+ mm_hash_overrides=mm_hash_overrides,
313
)
314
from vllm.platforms import current_platform
315
current_platform.validate_request(