@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 749be00a98
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

[Core][Multimodal] Allow passing `multi_modal_uuids` as multimodal identifiers. (#23394)

Signed-off-by: Roger Wang <hey@rogerw.io>

Roger Wang committed Aug 30, 2025 at 18:01 UTC 749be00a98eef8eab262cc3119893c00dbca22e9
10 files changed +455 -54
docs/features/multimodal_inputs.md
+35
@@ -13,6 +13,41 @@ To input multi-modal data, follow this schema in [vllm.inputs.PromptType][]:
13 - `prompt`: The prompt should follow the format that is documented on HuggingFace.
14 - `multi_modal_data`: This is a dictionary that follows the schema defined in [vllm.multimodal.inputs.MultiModalDataDict][].
15
16 +### Stable UUIDs for Caching (multi_modal_uuids)
17 +
18 +When using multi-modal inputs, vLLM normally hashes each media item by content to enable caching across requests. You can optionally pass `multi_modal_uuids` to provide your own stable IDs for each item so caching can reuse work across requests without rehashing the raw content.
19 +
20 +??? code
21 +
22 + ```python
23 + from vllm import LLM
24 + from PIL import Image
25 +
26 + # Qwen2.5-VL example with two images
27 + llm = LLM(model="Qwen/Qwen2.5-VL-3B-Instruct")
28 +
29 + prompt = "USER: <image><image>\nDescribe the differences.\nASSISTANT:"
30 + img_a = Image.open("/path/to/a.jpg")
31 + img_b = Image.open("/path/to/b.jpg")
32 +
33 + outputs = llm.generate({
34 + "prompt": prompt,
35 + "multi_modal_data": {"image": [img_a, img_b]},
36 + # Provide stable IDs for caching.
37 + # Requirements (matched by this example):
38 + # - Include every modality present in multi_modal_data.
39 + # - For lists, provide the same number of entries.
40 + # - Use None to fall back to content hashing for that item.
41 + "multi_modal_uuids": {"image": ["sku-1234-a", None]},
42 + })
43 +
44 + for o in outputs:
45 + print(o.outputs[0].text)
46 + ```
47 +
48 +!!! warning
49 + If both multimodal processor caching and prefix caching are disabled, user-provided `multi_modal_uuids` are ignored.
50 +
51 ### Image Inputs
52
53 You can pass a single image to the `'image'` field of the multi-modal dictionary, as shown in the following examples:
tests/v1/engine/test_processor_multi_modal_uuids.py new
+229
@@ -0,0 +1,229 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +import pytest
5 +
6 +from vllm.assets.image import ImageAsset
7 +from vllm.assets.video import VideoAsset
8 +from vllm.config import CacheConfig, DeviceConfig, ModelConfig, VllmConfig
9 +from vllm.platforms.interface import UnspecifiedPlatform
10 +from vllm.sampling_params import SamplingParams
11 +from vllm.v1.engine import processor as processor_mod
12 +from vllm.v1.engine.processor import Processor
13 +
14 +cherry_pil_image = ImageAsset("cherry_blossom").pil_image
15 +stop_pil_image = ImageAsset("stop_sign").pil_image
16 +baby_reading_np_ndarrays = VideoAsset("baby_reading").np_ndarrays
17 +
18 +
19 +# Mock processor for testing
20 +def _mk_processor(monkeypatch,
21 + *,
22 + mm_cache_gb: float = 4.0,
23 + enable_prefix_caching: bool = True) -> Processor:
24 + """
25 + Create a Processor instance with minimal configuration suitable for unit
26 + tests without accessing external resources.
27 + """
28 + monkeypatch.setattr(ModelConfig,
29 + "try_get_generation_config",
30 + lambda self: {},
31 + raising=True)
32 + monkeypatch.setattr(ModelConfig,
33 + "__post_init__",
34 + lambda self: None,
35 + raising=True)
36 + monkeypatch.setattr(UnspecifiedPlatform,
37 + "is_async_output_supported",
38 + classmethod(lambda cls, enforce_eager: True),
39 + raising=True)
40 + monkeypatch.setattr(
41 + ModelConfig,
42 + "verify_async_output_proc",
43 + lambda self, parallel_config, speculative_config, device_config: None,
44 + raising=True)
45 + monkeypatch.setattr(ModelConfig,
46 + "verify_with_parallel_config",
47 + lambda self, parallel_config: None,
48 + raising=True)
49 + monkeypatch.setattr(processor_mod,
50 + "processor_cache_from_config",
51 + lambda vllm_config, mm_registry: None,
52 + raising=True)
53 +
54 + monkeypatch.setattr(VllmConfig,
55 + "__post_init__",
56 + lambda self: None,
57 + raising=True)
58 +
59 + model_config = ModelConfig(
60 + skip_tokenizer_init=True,
61 + max_model_len=128,
62 + mm_processor_cache_gb=mm_cache_gb,
63 + generation_config="vllm",
64 + tokenizer="dummy",
65 + )
66 +
67 + # Minimal multimodal_config to satisfy references in
68 + # Processor.process_inputs.
69 + class _MockMMConfig:
70 +
71 + def __init__(self, gb: float):
72 + self.mm_processor_cache_gb = gb
73 +
74 + model_config.multimodal_config = _MockMMConfig(
75 + mm_cache_gb) # type: ignore[attr-defined]
76 + vllm_config = VllmConfig(
77 + model_config=model_config,
78 + cache_config=CacheConfig(enable_prefix_caching=enable_prefix_caching),
79 + device_config=DeviceConfig(device="cpu"),
80 + )
81 +
82 + # Pass tokenizer=None; InputPreprocessor handles None when
83 + # skip_tokenizer_init is True.
84 + return Processor(vllm_config, tokenizer=None) # type: ignore[arg-type]
85 +
86 +
87 +def test_multi_modal_uuids_length_mismatch_raises(monkeypatch):
88 + processor = _mk_processor(monkeypatch)
89 +
90 + prompt = {
91 + "prompt": "USER: <image>\nDescribe\nASSISTANT:",
92 + "multi_modal_data": {
93 + "image": [cherry_pil_image, stop_pil_image]
94 + },
95 + # Mismatch: 2 items but only 1 uuid provided
96 + "multi_modal_uuids": {
97 + "image": ["hash_cherry"]
98 + },
99 + }
100 +
101 + with pytest.raises(ValueError, match="must have same length as data"):
102 + processor.process_inputs(
103 + request_id="req-1",
104 + prompt=prompt, # type: ignore[arg-type]
105 + params=SamplingParams(),
106 + )
107 +
108 +
109 +def test_multi_modal_uuids_missing_modality_raises(monkeypatch):
110 + processor = _mk_processor(monkeypatch)
111 +
112 + prompt = {
113 + "prompt": "USER: <image><video>\nDescribe\nASSISTANT:",
114 + # Two modalities provided in data
115 + "multi_modal_data": {
116 + "image": [cherry_pil_image],
117 + "video": [baby_reading_np_ndarrays]
118 + },
119 + # Only image uuids provided; video missing should raise
120 + "multi_modal_uuids": {
121 + "image": ["hash_cherry"]
122 + },
123 + }
124 +
125 + with pytest.raises(ValueError,
126 + match="must be provided if multi_modal_data"):
127 + processor.process_inputs(
128 + request_id="req-2",
129 + prompt=prompt, # type: ignore[arg-type]
130 + params=SamplingParams(),
131 + )
132 +
133 +
134 +@pytest.mark.parametrize(
135 + "mm_cache_gb, enable_prefix_caching",
136 + [
137 + (4.0, True), # default behavior
138 + (4.0, False), # prefix caching disabled
139 + (0.0, True), # processor cache disabled
140 + ],
141 +)
142 +def test_multi_modal_uuids_accepts_none_and_passes_through(
143 + monkeypatch, mm_cache_gb: float, enable_prefix_caching: bool):
144 + processor = _mk_processor(monkeypatch,
145 + mm_cache_gb=mm_cache_gb,
146 + enable_prefix_caching=enable_prefix_caching)
147 +
148 + # Capture the overrides passed to InputPreprocessor.preprocess
149 + captured: dict[str, object] = {}
150 +
151 + def fake_preprocess(prompt,
152 + *,
153 + tokenization_kwargs=None,
154 + lora_request=None,
155 + mm_hash_overrides=None):
156 + captured["mm_hash_overrides"] = mm_hash_overrides
157 + # Minimal processed inputs for decoder-only flow
158 + return {"type": "token", "prompt_token_ids": [1]}
159 +
160 + # Monkeypatch only the bound preprocess method on this instance
161 + monkeypatch.setattr(processor.input_preprocessor,
162 + "preprocess",
163 + fake_preprocess,
164 + raising=True)
165 +
166 + # Use a consistent two-image scenario across all configurations
167 + mm_uuids = {"image": [None, "hash_stop"], "video": None}
168 + prompt = {
169 + "prompt": "USER: <image><image>\nTwo images\nASSISTANT:",
170 + "multi_modal_data": {
171 + "image": [cherry_pil_image, stop_pil_image],
172 + "video": baby_reading_np_ndarrays,
173 + },
174 + "multi_modal_uuids": mm_uuids,
175 + }
176 +
177 + processor.process_inputs(
178 + request_id="req-3",
179 + prompt=prompt, # type: ignore[arg-type]
180 + params=SamplingParams(),
181 + )
182 +
183 + assert captured["mm_hash_overrides"] == mm_uuids
184 +
185 +
186 +def test_multi_modal_uuids_ignored_when_caching_disabled(monkeypatch):
187 + # When both processor cache is 0 and prefix caching disabled, the
188 + # processor builds overrides from request id instead of using user UUIDs.
189 + processor = _mk_processor(monkeypatch,
190 + mm_cache_gb=0.0,
191 + enable_prefix_caching=False)
192 +
193 + captured: dict[str, object] = {}
194 +
195 + def fake_preprocess(prompt,
196 + *,
197 + tokenization_kwargs=None,
198 + lora_request=None,
199 + mm_hash_overrides=None):
200 + captured["mm_hash_overrides"] = mm_hash_overrides
201 + return {"type": "token", "prompt_token_ids": [1]}
202 +
203 + monkeypatch.setattr(processor.input_preprocessor,
204 + "preprocess",
205 + fake_preprocess,
206 + raising=True)
207 +
208 + request_id = "req-42"
209 + mm_uuids = {"image": ["hash_cherry", "hash_stop"], "video": "hash_video"}
210 + prompt = {
211 + "prompt": "USER: <image><image><video>\nDescribe\nASSISTANT:",
212 + "multi_modal_data": {
213 + "image": [cherry_pil_image, stop_pil_image],
214 + "video": baby_reading_np_ndarrays,
215 + },
216 + "multi_modal_uuids": mm_uuids,
217 + }
218 +
219 + processor.process_inputs(
220 + request_id=request_id,
221 + prompt=prompt, # type: ignore[arg-type]
222 + params=SamplingParams(),
223 + )
224 +
225 + # Expect request-id-based overrides are passed through
226 + assert captured["mm_hash_overrides"] == {
227 + "image": [f"{request_id}-image-0", f"{request_id}-image-1"],
228 + "video": [f"{request_id}-video-0"],
229 + }
vllm/entrypoints/openai/serving_engine.py
+1 -1
@@ -67,7 +67,7 @@ from vllm.inputs.parse import parse_and_batch_prompt
67 from vllm.logger import init_logger
68 from vllm.lora.request import LoRARequest
69 from vllm.multimodal import ( # noqa: F401 - Required to resolve Pydantic error in RequestProcessingMixin
70 - MultiModalDataDict)
70 + MultiModalDataDict, MultiModalUUIDDict)
71 from vllm.outputs import PoolingRequestOutput, RequestOutput
72 from vllm.pooling_params import PoolingParams
73 from vllm.sampling_params import BeamSearchParams, SamplingParams
vllm/inputs/data.py
+19 -1
@@ -7,7 +7,8 @@ import torch
7 from typing_extensions import NotRequired, TypedDict, TypeIs, TypeVar
8
9 if TYPE_CHECKING:
10 - from vllm.multimodal.inputs import MultiModalDataDict, MultiModalInputs
10 + from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalInputs,
11 + MultiModalUUIDDict)
12
13
14 class TextPrompt(TypedDict):
@@ -30,6 +31,15 @@ class TextPrompt(TypedDict):
31 to pass the mm_processor_kwargs to each of them.
32 """
33
34 + multi_modal_uuids: NotRequired["MultiModalUUIDDict"]
35 + """
36 + Optional user-specified UUIDs for multimodal items, mapped by modality.
37 + Lists must match the number of items per modality and may contain `None`.
38 + For `None` entries, the hasher will compute IDs automatically; non-None
39 + entries override the default hashes for caching, and MUST be unique per
40 + multimodal item.
41 + """
42 +
43 cache_salt: NotRequired[str]
44 """
45 Optional cache salt to be used for prefix caching.
@@ -59,6 +69,14 @@ class TokensPrompt(TypedDict):
69 to pass the mm_processor_kwargs to each of them.
70 """
71
72 + multi_modal_uuids: NotRequired["MultiModalUUIDDict"]
73 + """
74 + Optional user-specified UUIDs for multimodal items, mapped by modality.
75 + Lists must match the number of items per modality and may contain `None`.
76 + For `None` entries, the hasher will compute IDs automatically; non-None
77 + entries override the default hashes for caching.
78 + """
79 +
80 cache_salt: NotRequired[str]
81 """
82 Optional cache salt to be used for prefix caching.
vllm/inputs/preprocess.py
+29 -15
@@ -13,7 +13,7 @@ from vllm.lora.request import LoRARequest
13 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
14 from vllm.multimodal.cache import BaseMultiModalProcessorCache
15 from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalEncDecInputs,
16 - MultiModalInputs)
16 + MultiModalInputs, MultiModalUUIDDict)
17 from vllm.transformers_utils.tokenizer import AnyTokenizer
18 from vllm.transformers_utils.tokenizer_group import TokenizerGroup
19
@@ -258,7 +258,8 @@ class InputPreprocessor:
258 tokenization_kwargs: Optional[dict[str, Any]] = None,
259 lora_request: Optional[LoRARequest] = None,
260 *,
261 - mm_hash_overrides: Optional[dict[str, list[str]]] = None,
261 + mm_hash_overrides: Optional[Union[dict[str, list[str]],
262 + MultiModalUUIDDict]] = None,
263 ) -> MultiModalInputs:
264 """
265 Apply the model's multi-modal processor to a multi-modal prompt,
@@ -291,7 +292,8 @@ class InputPreprocessor:
292 tokenization_kwargs: Optional[dict[str, Any]] = None,
293 lora_request: Optional[LoRARequest] = None,
294 *,
294 - mm_hash_overrides: Optional[dict[str, list[str]]] = None,
295 + mm_hash_overrides: Optional[Union[dict[str, list[str]],
296 + MultiModalUUIDDict]] = None,
297 ) -> MultiModalInputs:
298 """
299 Async version of
@@ -368,7 +370,8 @@ class InputPreprocessor:
370 tokenization_kwargs: Optional[dict[str, Any]] = None,
371 lora_request: Optional[LoRARequest] = None,
372 *,
371 - mm_hash_overrides: Optional[dict[str, list[str]]] = None,
373 + mm_hash_overrides: Optional[Union[dict[str, list[str]],
374 + MultiModalUUIDDict]] = None,
375 ) -> Union[TokenInputs, MultiModalInputs]:
376 prompt_token_ids = self._truncate_inputs(
377 parsed_content["prompt_token_ids"], tokenization_kwargs)
@@ -397,7 +400,8 @@ class InputPreprocessor:
400 tokenization_kwargs: Optional[dict[str, Any]] = None,
401 lora_request: Optional[LoRARequest] = None,
402 *,
400 - mm_hash_overrides: Optional[dict[str, list[str]]] = None,
403 + mm_hash_overrides: Optional[Union[dict[str, list[str]],
404 + MultiModalUUIDDict]] = None,
405 ) -> Union[TokenInputs, MultiModalInputs]:
406 prompt_token_ids = self._truncate_inputs(
407 parsed_content["prompt_token_ids"], tokenization_kwargs)
@@ -426,7 +430,8 @@ class InputPreprocessor:
430 tokenization_kwargs: Optional[dict[str, Any]] = None,
431 lora_request: Optional[LoRARequest] = None,
432 *,
429 - mm_hash_overrides: Optional[dict[str, list[str]]] = None,
433 + mm_hash_overrides: Optional[Union[dict[str, list[str]],
434 + MultiModalUUIDDict]] = None,
435 ) -> Union[TokenInputs, MultiModalInputs]:
436 prompt_text = parsed_content["prompt"]
437
@@ -462,7 +467,8 @@ class InputPreprocessor:
467 tokenization_kwargs: Optional[dict[str, Any]] = None,
468 lora_request: Optional[LoRARequest] = None,
469 *,
465 - mm_hash_overrides: Optional[dict[str, list[str]]] = None,
470 + mm_hash_overrides: Optional[Union[dict[str, list[str]],
471 + MultiModalUUIDDict]] = None,
472 ) -> Union[TokenInputs, MultiModalInputs]:
473 prompt_text = parsed_content["prompt"]
474
@@ -498,7 +504,8 @@ class InputPreprocessor:
504 tokenization_kwargs: Optional[dict[str, Any]] = None,
505 lora_request: Optional[LoRARequest] = None,
506 *,
501 - mm_hash_overrides: Optional[dict[str, list[str]]] = None,
507 + mm_hash_overrides: Optional[Union[dict[str, list[str]],
508 + MultiModalUUIDDict]] = None,
509 ) -> SingletonInputs:
510 """
511 Extract the singleton inputs from a prompt.
@@ -545,7 +552,8 @@ class InputPreprocessor:
552 tokenization_kwargs: Optional[dict[str, Any]] = None,
553 lora_request: Optional[LoRARequest] = None,
554 *,
548 - mm_hash_overrides: Optional[dict[str, list[str]]] = None,
555 + mm_hash_overrides: Optional[Union[dict[str, list[str]],
556 + MultiModalUUIDDict]] = None,
557 ) -> SingletonInputs:
558 """
559 Async version of
@@ -684,7 +692,8 @@ class InputPreprocessor:
692 prompt: PromptType,
693 tokenization_kwargs: Optional[dict[str, Any]] = None,
694 *,
687 - mm_hash_overrides: Optional[dict[str, list[str]]] = None,
695 + mm_hash_overrides: Optional[Union[dict[str, list[str]],
696 + MultiModalUUIDDict]] = None,
697 ) -> EncoderDecoderInputs:
698 """
699 For encoder/decoder models only:
@@ -759,7 +768,8 @@ class InputPreprocessor:
768 prompt: PromptType,
769 tokenization_kwargs: Optional[dict[str, Any]] = None,
770 *,
762 - mm_hash_overrides: Optional[dict[str, list[str]]] = None,
771 + mm_hash_overrides: Optional[Union[dict[str, list[str]],
772 + MultiModalUUIDDict]] = None,
773 ) -> EncoderDecoderInputs:
774 """
775 Async version of
@@ -826,7 +836,8 @@ class InputPreprocessor:
836 tokenization_kwargs: Optional[dict[str, Any]] = None,
837 lora_request: Optional[LoRARequest] = None,
838 *,
829 - mm_hash_overrides: Optional[dict[str, list[str]]] = None,
839 + mm_hash_overrides: Optional[Union[dict[str, list[str]],
840 + MultiModalUUIDDict]] = None,
841 ) -> DecoderOnlyInputs:
842 """
843 For decoder-only models:
@@ -858,7 +869,8 @@ class InputPreprocessor:
869 tokenization_kwargs: Optional[dict[str, Any]] = None,
870 lora_request: Optional[LoRARequest] = None,
871 *,
861 - mm_hash_overrides: Optional[dict[str, list[str]]] = None,
872 + mm_hash_overrides: Optional[Union[dict[str, list[str]],
873 + MultiModalUUIDDict]] = None,
874 ) -> DecoderOnlyInputs:
875 """
876 Async version of
@@ -879,7 +891,8 @@ class InputPreprocessor:
891 tokenization_kwargs: Optional[dict[str, Any]] = None,
892 lora_request: Optional[LoRARequest] = None,
893 *,
882 - mm_hash_overrides: Optional[dict[str, list[str]]] = None,
894 + mm_hash_overrides: Optional[Union[dict[str, list[str]],
895 + MultiModalUUIDDict]] = None,
896 ) -> ProcessorInputs:
897 """Preprocess the input prompt."""
898 if self.model_config.is_encoder_decoder:
@@ -909,7 +922,8 @@ class InputPreprocessor:
922 tokenization_kwargs: Optional[dict[str, Any]] = None,
923 lora_request: Optional[LoRARequest] = None,
924 *,
912 - mm_hash_overrides: Optional[dict[str, list[str]]] = None,
925 + mm_hash_overrides: Optional[Union[dict[str, list[str]],
926 + MultiModalUUIDDict]] = None,
927 ) -> ProcessorInputs:
928 """
929 Async version of
vllm/multimodal/__init__.py
+3 -3
@@ -1,11 +1,11 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 from .base import MultiModalPlaceholderMap
4 -from .hasher import MultiModalHashDict, MultiModalHasher
4 +from .hasher import MultiModalHasher
5 from .inputs import (BatchedTensorInputs, ModalityData, MultiModalDataBuiltins,
6 MultiModalDataDict, MultiModalKwargs,
7 MultiModalKwargsItems, MultiModalPlaceholderDict,
8 - NestedTensors)
8 + MultiModalUUIDDict, NestedTensors)
9 from .registry import MultiModalRegistry
10
11 MULTIMODAL_REGISTRY = MultiModalRegistry()
@@ -23,12 +23,12 @@ __all__ = [
23 "ModalityData",
24 "MultiModalDataBuiltins",
25 "MultiModalDataDict",
26 - "MultiModalHashDict",
26 "MultiModalHasher",
27 "MultiModalKwargs",
28 "MultiModalKwargsItems",
29 "MultiModalPlaceholderDict",
30 "MultiModalPlaceholderMap",
31 + "MultiModalUUIDDict",
32 "NestedTensors",
33 "MULTIMODAL_REGISTRY",
34 "MultiModalRegistry",
vllm/multimodal/hasher.py
+1 -6
@@ -3,7 +3,7 @@
3
4 import pickle
5 import uuid
6 -from collections.abc import Iterable, Mapping
6 +from collections.abc import Iterable
7 from typing import Union
8
9 import numpy as np
@@ -16,11 +16,6 @@ from vllm.multimodal.image import convert_image_mode
16
17 logger = init_logger(__name__)
18
19 -MultiModalHashDict = Mapping[str, list[str]]
20 -"""
21 -A dictionary containing hashes for items in each modality.
22 -"""
23 -
19
20 class MultiModalHasher:
21
vllm/multimodal/inputs.py
+13 -2
@@ -22,7 +22,8 @@ if TYPE_CHECKING:
22 from PIL.Image import Image
23 from transformers.feature_extraction_utils import BatchFeature
24
25 - from .hasher import MultiModalHashDict
25 + from .processing import MultiModalHashes
26 +
27 else:
28 torch = LazyLoader("torch", globals(), "torch")
29
@@ -115,6 +116,16 @@ The built-in modalities are defined by
116 [`MultiModalDataBuiltins`][vllm.multimodal.inputs.MultiModalDataBuiltins].
117 """
118
119 +MultiModalUUIDDict: TypeAlias = Mapping[str, Union[list[Optional[str]], str]]
120 +"""
121 +A dictionary containing user-provided UUIDs for items in each modality.
122 +If a UUID for an item is not provided, its entry will be `None` and
123 +MultiModalHasher will compute a hash for the item.
124 +
125 +The UUID will be used to identify the item for all caching purposes
126 +(input processing caching, embedding caching, prefix caching, etc).
127 +"""
128 +
129
130 @dataclass(frozen=True)
131 class PlaceholderRange:
@@ -939,7 +950,7 @@ class MultiModalInputs(TypedDict):
950 mm_kwargs: MultiModalKwargsOptionalItems
951 """Keyword arguments to be directly passed to the model after batching."""
952
942 - mm_hashes: "MultiModalHashDict"
953 + mm_hashes: "MultiModalHashes"
954 """The hashes of the multi-modal data."""
955
956 mm_placeholders: "MultiModalPlaceholderDict"
vllm/multimodal/processing.py
+68 -23
@@ -24,7 +24,8 @@ from .hasher import MultiModalHasher
24 from .inputs import (MultiModalDataDict, MultiModalEncDecInputs,
25 MultiModalFieldConfig, MultiModalInputs,
26 MultiModalKwargsItem, MultiModalKwargsItems,
27 - MultiModalKwargsOptionalItems, PlaceholderRange)
27 + MultiModalKwargsOptionalItems, MultiModalUUIDDict,
28 + PlaceholderRange)
29 from .parse import (DictEmbeddingItems, EmbeddingItems, MultiModalDataItems,
30 MultiModalDataParser)
31
@@ -1021,7 +1022,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1022 mm_data: MultiModalDataDict,
1023 hf_processor_mm_kwargs: Mapping[str, object],
1024 *,
1024 - mm_hash_overrides: Optional[MultiModalHashes] = None,
1025 + mm_hash_overrides: Optional[Union[dict[str, list[str]],
1026 + MultiModalUUIDDict]] = None,
1027 ) -> MultiModalInputs:
1028 return self.apply(prompt,
1029 mm_data,
@@ -1361,24 +1363,62 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1363 mm_items: MultiModalDataItems,
1364 hf_processor_mm_kwargs: Mapping[str, object],
1365 tokenization_kwargs: Mapping[str, object],
1366 + *,
1367 + mm_hash_overrides: Optional[Union[dict[str, list[str]],
1368 + MultiModalUUIDDict]] = None,
1369 ) -> MultiModalHashes:
1370 """Create MM hashes to be returned (only used in V1).
1371
1372 +
1373 Note: When overrides are provided via callers of `apply`,
1374 `_hash_mm_items` will be bypassed and the overrides will be used.
1375 """
1376 model_id = self.info.model_id
1377
1372 - return {
1373 - modality: [
1374 - MultiModalHasher.hash_kwargs(model_id=model_id,
1375 - **{modality: item},
1376 - **hf_processor_mm_kwargs,
1377 - **tokenization_kwargs)
1378 - for item in items
1379 - ]
1380 - for modality, items in mm_items.items()
1381 - }
1378 + hashes: MultiModalHashes = {}
1379 + mm_hash_overrides = mm_hash_overrides or {}
1380 +
1381 + for modality, items in mm_items.items():
1382 + if modality in mm_hash_overrides:
1383 + mm_hashes = mm_hash_overrides[modality]
1384 + if isinstance(mm_hashes, str):
1385 + mm_hashes = [mm_hashes]
1386 +
1387 + # For None entries, compute a hash; otherwise, use provided ID.
1388 + computed: list[str] = []
1389 + for i, item in enumerate(items):
1390 + mm_hash = mm_hashes[i]
1391 +
1392 + # NOTE: Even if a mm_hash is provided, we still compute a
1393 + # hash if `hf_processor_mm_kwargs` or `tokenization_kwargs`
1394 + # are provided. This is because the processed multimodal
1395 + # inputs can be different depending on the processor kwargs.
1396 + if mm_hash is None or \
1397 + hf_processor_mm_kwargs or \
1398 + tokenization_kwargs:
1399 +
1400 + # NOTE: use provided hash string to hash with kwargs
1401 + # if available for better performance.
1402 + item = mm_hash if mm_hash is not None else item
1403 + computed.append(
1404 + MultiModalHasher.hash_kwargs(
1405 + model_id=model_id,
1406 + **{modality: item},
1407 + **hf_processor_mm_kwargs,
1408 + **tokenization_kwargs))
1409 + else:
1410 + computed.append(mm_hash)
1411 + hashes[modality] = computed
1412 + else:
1413 + hashes[modality] = [
1414 + MultiModalHasher.hash_kwargs(model_id=model_id,
1415 + **{modality: item},
1416 + **hf_processor_mm_kwargs,
1417 + **tokenization_kwargs)
1418 + for item in items
1419 + ]
1420 +
1421 + return hashes
1422
1423 def _get_cache_missing_items(
1424 self,
@@ -1474,7 +1514,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1514 hf_processor_mm_kwargs: Mapping[str, object],
1515 tokenization_kwargs: Mapping[str, object],
1516 *,
1477 - mm_hash_overrides: Optional[MultiModalHashes] = None,
1517 + mm_hash_overrides: Optional[Union[dict[str, list[str]],
1518 + MultiModalUUIDDict]] = None,
1519 ) -> tuple[list[int], MultiModalProcessingInfo, bool]:
1520 (
1521 prompt_ids,
@@ -1495,9 +1536,10 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1536 )
1537
1538 # Use overrides if provided; fallback to data-dependent hashing.
1498 - mm_hashes = (mm_hash_overrides if mm_hash_overrides is not None else
1499 - self._hash_mm_items(mm_data_items, hf_processor_mm_kwargs,
1500 - tokenization_kwargs))
1539 + mm_hashes = self._hash_mm_items(mm_data_items,
1540 + hf_processor_mm_kwargs,
1541 + tokenization_kwargs,
1542 + mm_hash_overrides=mm_hash_overrides)
1543
1544 mm_prompt_updates = self._get_mm_prompt_updates(
1545 mm_data_items,
@@ -1520,7 +1562,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1562 hf_processor_mm_kwargs: Mapping[str, object],
1563 tokenization_kwargs: Mapping[str, object],
1564 *,
1523 - mm_hash_overrides: Optional[MultiModalHashes] = None,
1565 + mm_hash_overrides: Optional[Union[dict[str, list[str]],
1566 + MultiModalUUIDDict]] = None,
1567 ) -> tuple[list[int], MultiModalProcessingInfo, bool]:
1568 """
1569 Apply the HF processor on the full prompt text,
@@ -1538,10 +1581,10 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1581 mm_hash_overrides=mm_hash_overrides,
1582 )
1583
1541 - # Use overrides if provided; fallback to data-dependent hashing.
1542 - mm_hashes = (mm_hash_overrides if mm_hash_overrides is not None else
1543 - self._hash_mm_items(mm_data_items, hf_processor_mm_kwargs,
1544 - tokenization_kwargs))
1584 + mm_hashes = self._hash_mm_items(mm_data_items,
1585 + hf_processor_mm_kwargs,
1586 + tokenization_kwargs,
1587 + mm_hash_overrides=mm_hash_overrides)
1588
1589 mm_missing_data_items = self._get_cache_missing_items(
1590 cache=cache,
@@ -1742,7 +1785,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1785 hf_processor_mm_kwargs: Mapping[str, object],
1786 tokenization_kwargs: Optional[Mapping[str, object]] = None,
1787 *,
1745 - mm_hash_overrides: Optional[dict[str, list[str]]] = None,
1788 + mm_hash_overrides: Optional[Union[dict[str, list[str]],
1789 + MultiModalUUIDDict]] = None,
1790 ) -> MultiModalInputs:
1791 """
1792 Process multi-modal inputs to be used in vLLM.
@@ -1857,7 +1901,8 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
1901 hf_processor_mm_kwargs: Mapping[str, object],
1902 tokenization_kwargs: Optional[Mapping[str, object]] = None,
1903 *,
1860 - mm_hash_overrides: Optional[MultiModalHashes] = None,
1904 + mm_hash_overrides: Optional[Union[dict[str, list[str]],
1905 + MultiModalUUIDDict]] = None,
1906 ) -> MultiModalEncDecInputs:
1907 """
1908 Process multi-modal inputs to be used in vLLM.
vllm/v1/engine/processor.py
+57 -3
@@ -150,6 +150,49 @@ class Processor:
150 self._validate_sampling_params(params, lora_request)
151 self._validate_supported_sampling_params(params)
152
153 + def _validate_multi_modal_uuids(self, prompt: PromptType) -> None:
154 + """
155 + Validate that user-provided multi_modal_uuids align with
156 + multi_modal_data in the incoming request prompt(s).
157 + Only checks lengths; `None` entries are allowed and will be
158 + auto-hashed downstream.
159 + """
160 +
161 + def _validate_single_prompt(single_prompt: Union[dict, str]) -> None:
162 + if not isinstance(single_prompt, dict):
163 + return
164 + mm_data = single_prompt.get("multi_modal_data")
165 + mm_uuids = single_prompt.get("multi_modal_uuids")
166 + if not mm_data or not mm_uuids:
167 + return
168 +
169 + for modality, items in mm_data.items():
170 + if modality in mm_uuids:
171 + data_len = len(items) if isinstance(items, list) else 1
172 + uuid_len = len(mm_uuids[modality]) if isinstance(
173 + mm_uuids[modality], list) else 1
174 + if uuid_len != data_len:
175 + raise ValueError(
176 + f"multi_modal_uuids for modality '{modality}' "
177 + "must have same length as data: got "
178 + f"{uuid_len} uuids vs "
179 + f"{data_len} items.")
180 + else:
181 + raise ValueError(
182 + f"multi_modal_uuids for modality '{modality}' must "
183 + "be provided if multi_modal_data is provided.")
184 +
185 + # Handle explicit encoder/decoder prompts or singleton prompt
186 + if isinstance(prompt, dict) and "encoder_prompt" in prompt:
187 + enc = prompt.get("encoder_prompt")
188 + dec = prompt.get("decoder_prompt")
189 + if enc is not None:
190 + _validate_single_prompt(enc)
191 + if dec is not None:
192 + _validate_single_prompt(dec)
193 + else:
194 + _validate_single_prompt(prompt) # type: ignore[arg-type]
195 +
196 def _validate_lora(self, lora_request: Optional[LoRARequest]) -> None:
197 if lora_request is not None and not self.lora_config:
198 raise ValueError(f"Got lora_request {lora_request} but LoRA is "
@@ -289,17 +332,27 @@ class Processor:
332 if arrival_time is None:
333 arrival_time = time.time()
334
292 - # Optionally generate multimodal hash overrides based on request id.
335 + # Optionally generate multimodal hash overrides to avoid hashing
336 + # multimodal data items by their content as their identifiers.
337 +
338 # NOTE: when users explicitly turn off BOTH prefix caching and input
339 # processing caching, no multimodal features or embeddings will be
295 - # reused across requests, therefore hashing is no longer necessary.
340 + # reused across requests, therefore identifying multimodal data items
341 + # by their content is no longer necessary, and we create uuids with
342 + # request id-modality-index as multimodal hash overrides.
343 if (self.model_config.multimodal_config and
344 self.model_config.multimodal_config.mm_processor_cache_gb == 0
345 and not self.cache_config.enable_prefix_caching):
346 mm_hash_overrides = self._maybe_build_mm_hash_overrides(
347 request_id, prompt)
348 else:
302 - mm_hash_overrides = None
349 + # Otherwise, use user-provided uuids as multimodal hash overrides
350 + # if provided.
351 + self._validate_multi_modal_uuids(prompt)
352 + if isinstance(prompt, dict):
353 + mm_hash_overrides = prompt.get("multi_modal_uuids")
354 + else:
355 + mm_hash_overrides = None
356
357 # Process inputs, which includes:
358 # 1. Tokenize text prompt, with LoRA request if one exists.
@@ -317,6 +370,7 @@ class Processor:
370 params=params,
371 processed_inputs=processed_inputs,
372 )
373 +
374 eos_token_id = self.input_preprocessor.get_eos_token_id(lora_request)
375
376 self._validate_model_inputs(processed_inputs, lora_request)